From f93d44d3013941e2c37a50c6c4b877d86550a457 Mon Sep 17 00:00:00 2001 From: Aurora_zxy <9166856+zombie233@user.noreply.gitee.com> Date: Sat, 10 Jul 2021 12:21:45 +0000 Subject: [PATCH 01/10] =?UTF-8?q?=E6=96=B0=E5=BB=BA=202019302110407-?= =?UTF-8?q?=E5=BC=A0=E7=BF=94=E5=AE=87-=E5=8A=A8=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.keep" | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/.keep" diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/.keep" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/.keep" new file mode 100644 index 0000000..e69de29 -- Gitee From ef218c87e40f1550decb1dfc4dad3ab6ea389d78 Mon Sep 17 00:00:00 2001 From: Aurora_zxy <9166856+zombie233@user.noreply.gitee.com> Date: Sat, 10 Jul 2021 12:26:27 +0000 Subject: [PATCH 02/10] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20tools?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tools/.keep" | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/.keep" diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/.keep" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/.keep" new file mode 100644 index 0000000..e69de29 -- Gitee From 7ee4c5b54ff251c1d8b815557238b0607e760b7f Mon Sep 17 00:00:00 2001 From: Aurora_zxy <9166856+zombie233@user.noreply.gitee.com> Date: Sat, 10 Jul 2021 12:27:00 +0000 Subject: [PATCH 03/10] tools --- .../__pycache__/demo_net.cpython-37.pyc" | Bin 0 -> 3087 bytes .../__pycache__/test_net.cpython-37.pyc" | Bin 0 -> 4780 bytes .../__pycache__/train_net.cpython-37.pyc" | Bin 0 -> 11571 bytes .../__pycache__/visualization.cpython-37.pyc" | Bin 0 -> 8318 bytes .../tools/benchmark.py" | 25 + .../tools/demo_net.py" | 119 ++++ .../tools/run_net.py" | 49 ++ .../tools/ss.py" | 19 + .../tools/test_net.py" | 202 +++++++ .../tools/train_net.py" | 524 ++++++++++++++++++ .../tools/visualization.py" | 347 ++++++++++++ 11 files changed, 1285 insertions(+) create mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/demo_net.cpython-37.pyc" create mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/test_net.cpython-37.pyc" create mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/train_net.cpython-37.pyc" create mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/visualization.cpython-37.pyc" create mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/benchmark.py" create mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/demo_net.py" create mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/run_net.py" create mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/ss.py" create mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/test_net.py" create mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/train_net.py" create mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/visualization.py" diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/demo_net.cpython-37.pyc" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/demo_net.cpython-37.pyc" new file mode 100644 index 0000000000000000000000000000000000000000..4fca089fd42266b3d0389b24d012d1de31e24bdb GIT binary patch literal 3087 zcmcguOK&5`5uO*x;X{;Gt6eK{98WfD?-EF)_>zZgIH5#pEm$H6l9JW~2cywUlN@TM zhuhO!JxpDqTpjEo$T6Uf34$DQ2=W{J6Z&T45Nv*e;e)GsBTRgRji$x*j;IU2SBqZVeOtesU+ zRu6NLX`4~r&da$G7NVkElyoL6MRWFCRJO}hU0^u-6Yf z9y4S~kypAs4Q~Y>8Wo!dE%~>qC|nt;Q*ELQ^($3XxG~Ml=5lS)9pIJ`StlCyiVzdyk{vS`WzPX|t ztI7ZP13!PM{vK|5k(;Q~pIj-ba>^%aP5E8;xprY*+xo<--vxuS+~5H_JV%0YSjpLgD|l?ms&@}>cwN~ zTLGQz!@WrupY+^>ucf=zeA08rAx~CEXEV7Tb0gx6So|CUmRXMjPMC`!xo)-2_#md% z4aedGms^bRF{2iDlVdAB8WN8u>4MAP7C{&if2JGnZ~{4EQR0y#aoHK%uotsPzSc^{ z-oSDb>rmbyIsB=082E>l3lBM@pl5i;4CUHhCAJ`clkDOzX;gH>lT?EU* z)zZl#06-MebaT2#m|h&>Lr21h&~vJtr_M>tjtNU%;*o$9I`~%PT?85jZvGN8{`|uY z`D!ZxfI;Wdn4^FDfGOY^G&QvbZrHQpo|TYiV?sS*g@pF`V733n(m$Sjeg3E4R4PKF zBavlb%QzB7LWnQScH@cDsn=`ph?sc;QH)#)Nao-w!^6c~0^S`t*Dxpw&Ao1Guj|yd z+ailhfHDI;D0Dbc7`6ItQ_SyFH|jgi*7i=_X;r(Qh{et3?ryW;Z0=M$9j8&x{)rWD@Q&E({(-1STON zBcVfO5PF|*QP`$FIhB%?0-ML<%D0oxF25c~5M_DH$rxjUEuee`Ln#zF@Vf&JuuTkE z%tX879)-l=v4f=tZUDT$Dr_A#ee-B-5JzOK0};CgSn#t5I>Od?9ES-YLKHica6klM zvN3g#aSyg7pF^h<%W6q0X$7^QFRO;Gsag1$ve#fN=?kz{28jiI5x9nCs%0286?Q;x z!Yay{Fk3)Pug%y;px?txz%K(W3vClx@@MFvBtV&C4`I}Xy01dHddQ);T&fr9M7;!< zrj%!<8Gv*wOWH8YKITyTU|pvsH&g2L0nbnKya1&~ztpaC@uGkfq}30jx|* zSYTkB!;zP#^S(A+NVQM2Whk8}D&YxQn@v%B4Fh-|%4-PozKcfjbP zC=<@vXg-CUMW3(_kUfXlRjoPKqUb1=!v%vs^CRg=k@JIO6uM`^7{&n=880L*Wh*H9 zHi~9b+^=M0=-YPzxo`eAIY;rS4Dz3&^e?dcHFg-jY4;&e!ob)(1S(mEPBE7-vLNCh zx-?U}4{?Yzdje~fWs!m07@e_2n8`=5`$%m-XP0m4;_8?~36;=izdvK%Dk7^XJYUzD zm`R+Fm|}nMrX9~+nMP1lZ#|JD(xmo-H|&$S7{)G?VKy_dJuBj?x9|1pMhIW_qc

zA;|S#))a{g07w$$0nklHmMq6forP&BZTOn`r6Z&d%#Q$q#v%C_QC0c@Z4N)kZOCvP L8svFFH$VLj7BX?m literal 0 HcmV?d00001 diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/test_net.cpython-37.pyc" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/test_net.cpython-37.pyc" new file mode 100644 index 0000000000000000000000000000000000000000..69cf893a15dc452969671bca21d9a3615ca926f6 GIT binary patch literal 4780 zcmcInOK%(36`mL06!owyTZ(K?oW!vbTW*^+1&p9DZPB&>S#m`>E(Q*!BhHW>Y97p; zOIdb@Kv88`w?=+IL2d$c(M{1+|3ZN-y6EC=3$)0lKOn1q=MJfdk!>l=eVlvldEN8U z-IQe4&50@c#wtXsOA4a<;o&dOodgM3)9 z3TlR13`?4#Dq-2GOd!<>WX7sZAhQ$5oHaLr%ugT-*1`mGE?ks!&Ra_oVgwh$i`GTC z&IQZiinSuw`QV*!)mjZNS(jABn;TvBu8dX7^m3zj$BJ8V3vTf%^+>g@x+S;#m14c; zYE5OW`s(JRLBRc+2fp{hG7HG?5J4ZC|Tf|qwcMo5(q$^6Q zj+8NMmTDuF<~Q2m2G>&MNZr8ti%*|v!`w4tm>(8~#bGJcdpi3!H_|-Mb<`t`1&5#V zJTIjABj~xId~tDDPV;GbS?Lw&m0qi~`%L+A`>7ID)BF@V%%_D>ajXmrX;IF_!)Iw} zR2nOw@fbWS!z#E`CtUt#YBW*Z?m9eZ0YnXGGXGu?BeU#3OX2!~>Hddm1wgzr?c0aA1I+JVM z&(8MJ*;8k7ozZmj`v&O#na}p-`21)gof)g-C9fgFF|2kBs~y8;k72XW<=o`O7Soxu zmd=iK`F`i!QkolUsuCTLc1LO^tK_LM1^kwP4~uv8vHDCO&d{4mi8W?IJ7|d;1v%5Z z(7t$<8)*d$8jCbeIfJh%WspAfjKT@^F5_^>^u<u+>c=9hYxM^^wdGx%5=EksZPNZMU?1~tz zIo}{S$|ipIWA;{D-j?vk9_z#`G{XT8PQ|hxbxr=<;pQOn={$I8CQjH7yu{oF9lxKL z9TtbjA@2ln)RhE%o*)~9{g^p{=^XmWy5#LS?dS614b$f)clJT)6SK`?P~u1#@qx#{ z(K8by2j2-KE!PjdC_xUnVRqsmh+lZFx%bj^fa`QK5pf~z@X$FllXlEN)B()|QxKTj zho2xLc`t~Aq$f3*|_BA?#Ab5GR z9Y+UoFd*ShQA9?SY=D9{a6;((!spN90hi{?XxCXc$w<=UlQ4FcY0c}CjC*^c>QvAS zoZY57G7+2CvwVD8vV>JQbNmQ)$jp@C;Q5Hi$yLcNq~vX0Z%vc;8fZ*xB9G*=!yM9X zcx5l#3p~kYf+lT5lF$z?i(-?U@Mw3s=Jk7>?ly`GNW)YezYAa6@VMjSHQG*O?%^@$ zolHMELo9(Ob)1CX%5L3qy^bTP_g^04y$4pWL=8(pl}yI=MD7l_#nhEHjl=>IX5m zNp)eg8qJoNZ`MC=*t?D9ldV?Meq3)o6k41JgBllM^kP2}<(((3$4^@J#^$c5`iaeA zkm$3x%``lVFycOJk?;HM{Qy>S2VtL@9YjA9h97lek*93raBEH`4F8p9<8_(ePSzGh zMTSrok|HO=Oq3-|#ttmj>56>jPFAr!U{ZIH_oM!RC#*&ilrj~4_fQz*H$wG=zVE#h zI`m{SlP%;|fO2J|rV5o%uJ!bhkt#je4)FY_aJ(*#N?4aAR%KLmWy$4OO>WLm%Q;j* zt2S0x54f|e$LD(UXk`>vbM@C~a`?jNT#A;J3OQdKoyUoR;;#`MH#b`17gFsAlw~q9--LxXY!jRoC!SB_T~l_f>q z`LDe5-uGdwC}(eA_kQn#DfQm9-iLr(>s>#}4|S~8ddp}7Ez%Y{lJBEMSWv<~9^D|D z42{Tei$jCoOf}d5w^HLMhueUnMN-+XQ)9GF4U1bu!%}+nCz&%(x{@>hVNgMxlt1|D z_tc^+m7Xnr|IbSCYQ6>6mi2d1f^OeOMxGrco-M_*Wkd>tBH+~(LM9!ux{3Hl(nUK& z`4u%jnI?3D&bQZYF%u3dmi@@*wu>~&{JjD9+>A0+2rWWVLF#qmP#B5lxuUqc{lIQE z8XN2~-K~U9G;ji2o)swl(6yPwR>!=2-yPOA+Jzr~kO=2D837OOfXx-!yPO`Sv1LvlnaTvZqK zygCQCh7>p_6JZ{!vLUfFL&upa>8+{TYwwGKZM$*XwnYwx>%U|VK%ZTwVVwr)JLmnF z%H2A8-w@q##}7Q)kF)BwJZ+)Y(VYg#`pHX+oP>z-35e~2rdWCF$|;8yl~bF{>6hNZ zp8Dz*kvjL*A^9SEj1%kH@yUUo!14W8GP2hB$&0{E5;1e;*qVLooH;~n{+QUr&63QG zXOH31^%EH}*7;*-W)fMb6X~a!ua&zV`MvA_inbl0bGL07ccKu*c9%J>^b`3^r(EF< pWMSkFqWBpY4g&9I6r#yh492t$s#R?PH8lIHqL$8SWw=B&`5zLUfQ|qF literal 0 HcmV?d00001 diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/train_net.cpython-37.pyc" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/train_net.cpython-37.pyc" new file mode 100644 index 0000000000000000000000000000000000000000..33ca069eb951b201753c8af629bfaa6ddeed8081 GIT binary patch literal 11571 zcmd^FOKe=%dA_fCKS_@Gk}O}#w##nUIog=0u%*c`Bq# zhm@IeA*7rMDYNC-kTMri=F9UTWg(<2mKQ_H+2%RH`Kj`GK`|OnH_fsso+FKCnit9! z#B;RqZ1cJDbK*JHc)oeDe6jgL`Gw|Ed8t_}7X>}u_(Jo=@{6kC&JHfQFP*66W#^2O zaFQRYrzV~d2 z*|nBwoA+yu+cql=+w*D%wTkW6+AXu$cHGADC$tIMMU7{xu5azPEReem*WI83w^1~BeD(I~4cB)WPi}PDmBUi4NzY;& zk{-WpH^dVfPu=J?{92XOoZZTyOWjstG#q@+;g`Zs$ItsWq=7n6PO!~^K2Yi3*8=5O z?Z<*hU9TH#%a4ess7NI>E}j!2O(IRz(~kNm+K)fhj~@CNYQL^L*5B>j)%uCMMnBn4 z_0#gc^7GnhG1(DoL3%k}f%o|oLwSMunx;Gdu1pFhPPIq?72 zJIeb%3-Uw$K_tTLwrkrS;y1^Qf|1yd)QSh;zzf?~zc zYb%HcNyk`Fg4D5wQBC(#!8B2>pmlwrZlKP?faaEQA`8l~_Oa56Q~p@}Sn(GJXT_SH zJ5ime6MLlgXM&lBf>*!bKQ%ZX6i!t1stukNy%q+h_}2!{OpHQf2mXcnvq8#<;_cUK+v*4p=%2P?F~PX0vipZ!QVP{qZM2WQuDmPZ`rxOMinNqE9jUV1eckEK;f~8fGKPFe9o@BbR4m@Uml^sQxQ*e8*Zr`RsX8D`)2W z#*=f!8oes^TOWKeFoHyQ62%>U|D8K@pI-~m1J+I-yiPTp6ogm7wlzgbn)uvw*+H8% zP5;mh1({D`t=6iVpquTMYq}(BmPPGVR`o86Cu%EdT2gG8OQHC>{3IJmm+h&QkY?tR z6l#|zI+g`~Tk;(@Xm|Wtv(|H&xg;KoveB4~CY^8^AyLX;h+&$R;QN9%8a550V>7!+ zqSPDrBc~&%=F(WGQg_28kxfi=*l1S;^HEVa0=rdl$1E#dW{Kr9muf8^oIqf1%a?0v zwhuNcc4wP?2vYE^I9Tz7rD*6_@V-7@!G>`u4kn6=iaX?czI2M4z2 zugF#_j(cE33wX<&2cvo)uo~u#8M-a6&Gy?ibM6SzC6iXK-Lf0#_EV*1d%x~h{1F>k zMXW=}ZK66Yjhsy}T23`|{(657@EO3-WRR=Z{GH`IZNSi@#j*YkN4llNU`yNW{bDQZ2bN>>_f&+Em-30P^6LQYj1?R^`A z^4(5R+lE4C1+@7jf}}#+=|f$Sf}O$7yMUys1j@UMebs?_g}4NsI#JIm^NNFO{$1^T z0~b*bd;j+@i;0A}uYCo58Xk^3kb}1TGFOj4h;%S5@6VK9EUg^2o9@ak_WBz3`lTik zm#z5icEei{=h14pKDPW9$t#VT=U3Xz&YRz%QPB{iO7Ce|N#5%#SV_C#y}3NDTQoj< zC3;!8CZ%@g70YF8BL6z&i*N9VV9n!n#4Nv7;d;Yu0nlu|vv-q6H?~)=Zf!7HHXd2s zx_Nz-r`PsMD79qHxOr>mDo^d+*(|MHw?t-oYxU~JmbH6hXQy=Cy1lx!w*h`T>l<4< zwY_)4TH9LP-QC#b2^Z(YWdKz?PV~2K?(Onq&8xKluy854L2JPy%x+bYsdOEiM=;E$ z$0I^&l+%RK#G+UY#v=z1Tz)y~frxAzaLu;-c7-R_H%c38rOlmff^44WdgTnYq-X1( z+d_@4o!wpQ+TQjWj%3ZN@H_xm<(}2DTcQ?@Y69Y{ZB?1=@c6!6x%UBXA+{Xn3rpB$ zl|yd$?anGGuAXjtN;V>&buo(@5!K$JXXJC5aHZ*7LF zvA$Yb<*6HcTcu4)tSFB`n6X;L;~DD7vnp+fri#xNX?OIdeKg!f%sGoYZHYaVymOS7 zOC<>r=3=3W=XhG4Eh+SQM95yA7|Cj$lI22>^2A6!a=mg;!$T2HF1hEZ3dG;FrKp*bo;cZ3s=;5RbK^p0p!i2Wam$fS{tV z17iJnKLPA+)OGd`J`jx`5mrD0RzTE`1+fuIPXXacK<_Q^%zi`F0s4Bb7Z*?n(yameyFt!!Ji?V9AxPqu*&ZH z*?P{;52hXhi90bT{s^|fkQQWzw0?F(%Z+F`j0-jp@SQdw`vCk;bYS3rrqPilfpHY- zXVA+}X+)w&jn&cHENaKA8jve`>&II^_U8sb^#Lpe;NQH!zey+cVLZr(_;;#5J;6f* zj|Je^Y00BcrBj1Npx$Xpfq98CF*>Rn#GKfIGC1qRfO^kKZUXgU%t6FS1KneEK*LW_ zZ9@0hGobH42f~jI5k9z{r;(nJ-31FM(1U3BERpXyv0r8YTt4Efvsv^qcwXS{G}QeWdk!jCK&TB!oi3Es7@-1dOE7FVOvq9Rp-uyCwD~mLCRqv0MZN<9 zWI$BpkT6lVh9dGsP>nIZlrVdQzelLsogC3;qFL%*2yS!f_R!C8*_4!_nx!E?jkK9# z+Q6q!Kb3A{G*78<#$y@D{~F z&%HFx&q`?sBZoN&B2_jIX1y3yX|CQpH81jI?owA&Md0gvUzN@)QaTb5<;DMFC#(&|!%kYC)5)cACq-R?p5Labv_r*<+@Qn2$Q)uXQL;?Q z3MDU7@(Lx-Qt~P#U!>#>O1?zN^OVpa*q4!D&bH6i>8U{2w`xO1FsloLwN`CmU$?2| zTa;{40!O$aQL?PbzDm?BN;W9DMhT%^c7u}ZNceOJq3N8|gbC@f9V)#^rSXF<`fb`B zo}d8;gG}}iBGE*r(S{8{Q~oV9D4yX-Jclxy(aGfrA_$quZWH4>l)OX9*C;7dA{Ov2 zJ-tiGdz4s|e4UayCHE*fq~w4SvK-lcN~~J!{NTYoN@yCQ;-FK@t+x(v>BA3?LH!zA%eukt}p{q?FG7jk(h*I%PBp)dssRtTizz^en!-p{)7{}17$Iz~h z4y|g~Z{rLKt-2??>Oyx2y9E9bbH8Oa><94lnEMZ&&_~$7O3iakQTMYHZm3qrzOBzu zw!)|(pV|LR)yhLAhKbQl^iEC{GAPK*@jk5+i7ZsQM{1Vr`YsRz>Dm95W7l)7YNu;1 z?ZZU?-_8ezF5MjBfg%%@vI$pTs~uvF3Km^_XbWgN=vI0TlB4$?l9T=Rdum_ls}EIn z8{47yssQkB1Hfy(PH-)B<0FU=z@1#D$Pxi~?H$_QByc0POqGJBUt^Ph@+j z35$`>l@t&);--j8LiYn=icSrWg_9l2cb>AUE*aj0klrL|l|@BG9>MV7_g)(T6${I0 zjimsc4@7Ckifn?}HgqAa5ynYc$-=Rd@M>#@CCB?&^{w1BhMuR2JN-~>b zzlZ;kZi*ul!*2~g?;#S}2w+y#R|ndO!Y&31?5m5uQIFuhh)^D(Q9n9}Q6A9?&5sWf zDADn(htCF{4L@ENbZlDWk=~E^Nk0|nz~G2c^rL$Z#V3%6zA{T))d&6YlR2NVi(cH$+qqH)|vZMA;bUjwWO-?F_xys6HZ(YfidP z%%yeAd`pG}CzwwNp>e~po_V4f3dj&>Llk~y!%zrPC~_$7erdNNHX!8>_DgIMi7xsK zW$M-K@z^GOK2LhqvGsx(T(@4A(dLiV@Cms%(wnEO4>~(fFbF41md^wWavkSLILN^W z7ZQ60H%BN{6lQ83Y4q^I7c-1J9TIM2Q6#;r@sRfADP&9 ztJXs7Rr zVvh=Yl#r~MkbW5%Ll(1k*%pZuLSy89%Bp!i4Mz~Ug3>yavOaVJ(LehbwMGJG8W}Q7 zguwZGNIhQ}kRVYhSX|c!1_Vw-XW` znt;j?I#ug=h$agDQs`8Hu$mf7Q>YBR;mMyF6sU#BpAmvj$9QHT_3e zNpT`hR0uu_s3I=}+_}M1&_xhiaCAic^Kc+2a7d7Flh1H0z^$az&H6L!Km7A`Q4UUt zM|ulDq#s8a!m4npT=JjupAVuW4&k^-LR=;UB_=3wK}ojG`4@+^AVL%V3&@EY$R)vR zN$_Gc1LTVMeF1?$75th1;^2~4n^X&0w4V~|kQ%&1&XT#|3M^w4;np}>@>d{;mHNx| zSB^E|bSog>F4h!Ik1JSbtZI-sk!zX_vPAV?rS-cT>+5}^A04{q{BsB>DztN`lSS?| znd|4A40_KXWSqMfeXM!2{_EdV5x7?BZ`8lk&x!G6Q3Eb2u{-AmzZT?ZJfGkE@T_&oaqj8&c*XHL|dmmK7doRypwCOAVKs2J=*fTHUnAvu41@${ zAW_!w;(+dNh&tCWia1Jvq_BRdaZIZ>KX{8=TYZgqvL9l1uA@9DRu5|@Lda7%JwNg{ z;iA;(6$7!NQB`@YzW*aQ$5I$S1=4?xS18^V?kN?j`)u#0LX?+Iz1Skv90h}$cI$zR zTzbM;C91(yC6qE57{Z}d^QjQI&+}saDwXv1&^pCV`E*~-zgk`F05Bk7DEWAO3gAj0f^8Hv&o&k(#^e+ zwZ3_aCq%sIty`PxJh`<4UTa@l+uA^U@A}TIlFTrA52>6LqiPOcSSnA202~gSs#SSV zX(04P9U%B)kaFeGP6uGY=dlhBW)p4!p{$Q-V?U&XG&1`ZC8UPgA0XjcrOVHdt_>l8 z2+DPUi+l;+KN6L$Zu6E%Y_1YMv@>Z13*ey1KJ^ zYh8Fac#=YpyVsH5<(j|GrzQgAGTP3g;m8Tf3qJ^t?(S^e-k@b$eaG6kxwCd%=wO~b zv=N@85TY0@pAq3kM13cT*;Rxh8TorR7;7VbS-Q2lxy=h}*EiO_dUI!UyJTJ4+`_z8 zOV<%(r)8L6nez_22L}z;B8Z3$ML3lXHHC0R*k_an`ay&qT@TBfF3;NchtWX1sp0Y| zd4?ye%m6`+yOSk6?tw%vrU~pd_E0bw5Q?yO5QnU__zW%P)m4OqH+F~n!e{MjmAUZj zxR%%5AH7J&a~>|$L|?+kAbf$ITbve{s+<^|G$stl5ys*vAVwBWZVb2>Sy*fgvB+Ih zPK{T930Dq}RSqdw>x3&bo^Lrfn!cPm`%7YY8mS41)FXWM$0$_di}##@##bsm*j_G?sW z;mJy4FfE`lU2;CI)b;VA%<_2sc|=fna!g|1qb7?ZzOu=LT;!>VjFg?@p~~8!$Yb9n zR!$j$Qp^i^8AxkhjLs9cb=4Q zM+oU1x9}E3)3q%2C+I|qmRL)^2_bH(xb{NqNew-|BjQQ;P2yKDP>78;$v5JW7%UcJ4n?Bt=owkEtKac3b`#%VSF(&+hy<#vrssdxXf6SCXxn&U!g5R*NFl zY*N)tTh=x=i_z?50dJf|kX#l4dCWCH_7p5~-UN#vxfM329D)II%5g5q_o`cz_9i2bK&(Erg!W5>uilxe@ zW@+-NTe^H2mVu|{CK^dAsYbX1jg+RSN+WG$V#r`4Eg@NJD2C)>$gq`|m!+h%yBUsc*t#bzxW8<(M{g;s*kzDYL%eY z^q$uIIZYHEI>Dw>tL_9vj}<+L*PgRv56{=zKQ0t^Zoh5hEycOe9FV)TTlukO->w{sM4X_kTztTaI z1m1UK{)4!DM$bRC?;7^4fT2eBbEgI>ViOX9SeWYnA_V zFcs#4>238;!^qnIAUGS$gtN4AO$Bo1j^LeK zINUu4Z4B>kg(KbbJq6THq0O<*I5>>=IsD!JUCCkm1c&ic959j`egO{oaQuivI1-LY z`2Hqo>_UAE^8P2zz=TXAY-CF5(dnJ(6uevn?WxhYucNWo8o$#YUBqZU#2&(7HY!g( z2ihz_4v7*YsUwa5Hkj>R3UfWRlaswO=$(v#*G>~KC@{Z)>-N>;sa#WC`f>#K0RW&?= zU!kh%il#tnXn-ZFYH)>a*`bD8KN;&w=eOjJ@RZU=T2nidtRPpNl&elgHyzqdv9Zq7 zYjywEkUn0GZpCZ$*)NaoMQ1uZL#Nj{8&1>i>Q|}l%!CCAJr_>O=XtF1qu^{f6Vtak z7s4T?zOQtK!YTQjiqGY(*Si1raeowNZ1CTLb~3mYPRFzvoQ>JJ2%Ne4bvCi90ms1T z?fMPbyBJ<3>yT#FOND?^@8rtPaB6PS|>viFS6Y%t$l2nnbEe)V)v z=Ox~l#8U|`JP%ut0=Lgt(i^1_Oa4T|N@%w=@VSIF=!P2n^YEOP49`jKm%~i`7S_KU zUSeloVddbR?(N{+?svl3p4w9(%NQhqJUyfPT}Y&YlU{!>oMkiJ_rsykWae|+8&FsYhJ4z_*e;$BT7fE80|(NL zA~>dZj)^MURrA{Y?dl5r(%c=h()6}#Re#=GbOO8P`ew!U%q_>n|K#ph&>h z%dawb-I^a%nvK>+MPf~IB-Mi}yzP}wBj0tK72EYcnm?wPGpr%(zUl-~@Zei%3LMG} z$M&q0?D_!{sY=sra_shC;z7-;`8y`#_KTWVHRG#4|K)$-vjGKXy{1>*0;gR}i^NuQ z-|+!Xy7rdi`kb`&yHysdU8NVROzH$H=#^G`PUCqjAauM^#|q=4fblUjZTuny{C-UM z6c#vQV&HzTZ{qU&f0K(24Sf@n&M_0A+t2MU>0IZ(q;Q+mxg`~QFt$Xt#r$Ds;V}Zt z7xqhZ=Ttrftb;;-?l@NT@0fGCoEGUVJE-iGYiwT($&eA}Ri=1z?n9AUdGe_I@bQy% zk)huD$Hle9b)LeiVsN9hvc9@@ZxtYsezdw+S}s3b0^)M<)6!abd1-xvLfmTA;likS z+f6<|WJ3|=W@+i+#~bTnkZxRjcf?G~ZT3fHSjqAUJId2Uo{;2Xkfgc4TwGr-t&6i+8T>e{2?W088eR$MG2uolVEO7Y%ui6h!o zgt1uMC<^0IskkEa^^HYg1beI@GBv;E`GM_K9AVI#B@*2Bs*ccI#}h`S&1{i?0c-do zy;#~P-QQSRT@k~ypA%Ax+>`ZEd1LKKsk~ggS6UVtYm0H)b<0(p6N2g4JdW7v?|h$*KRo64j|1&3A!C1vDZr*LN7io3jN;0H8Id~ zc%u!BjC^6+0j#u`;LdZ02jxJzma<2$FJ%BCO|Ndb;?`Os5j1&aM`X5nvrz`0)=Qyz z0Nb3~FGafT`Fm~0AsU-87!x=gk;I8M>_7|_vGb=z+Wq?66_GjiQ6eF|3trgmsz^rm zS){!(bDr0brif%@cSR~P#V`u9P=S?}RAjQmP&9LNKhlyj%~&H<` z0$&)!N}F;jyk^!gbx)94IRZrQkq_NQxN@neD_>G;=okxco0XzmkfgNm-eFmp8{*X! z3As;hk_2*PS}95Z->?v3-IsG3p@HLGiC65o`X)lyng z9o4g1R@LN};CU?rtQj?><0#!~csFUD0u7eA(4RzeeYqJKhG?Fl{X4NEm8fFT57Oe(KP33j$sX}&- zK4jU*4#~9j7w{&55^79+rFL`#z;2*+RV1rOA7m=0KKle=Fr{+|N?1Ds6xi|~2S%6( z5|qqwcmKU084QGpIN_K?3Y!R12+>mvsV`DFjP5FXD>9Kw96>vYFxgEZkxMdzC6Js( zNog91Csso03O)#a4K9LoHxp*UWL1-iR;H5#_jJtt*K#(E5H{`Qqka{VvjGIhX<7s6 z#cgmJBu-&QJ~PMkn7zJ#OUj>)C-athIB~R2Wj^#84j}K zgmS1M8NJ-eB0wGjCGpGklhE&3*d2lbZ`q^gKT+OYM+AP*w&DtMTq#dSdmi^PzG>_ z8V4hrEKgBdSze{ti)aq6-FI!@M{0lt9sgU7_zohKmAL{>650FIJfP+hH6Kv(AvJfX z`2%Y1Qd2}DGRJOII!{ivz#4+oQH_htwn#D<{K^jBC9-SO%u(|MjYuNb3LIt?qMcVr z+7tU1&?Um(xgowuB%e@22V|Z7CQZe4HMw*C8R3%8;WjlDG*&jc>F~HXph!!93^<&% z^oQgK-Z&}*ME}Fl{0hYWJv2%>qh5x|%)-P@lUdetFzy+QvM}>`^dloYs!nPt7}RlT zQZ*Rvf^MLn)A=4&>EAbs(8}v=t(<5kLbQ4rFLoF{^ zALynq2UCU-3{}6M?GB-jsu1AA-8_>nW1sIRAM-6$Uj# z)flE&$Ls`E=|Z)K%s!avPGiRCX&o5+`q{e5e}m@?e&_1vcQs#*Sb}=Nh0j%5Ii5Gi zAvNlR7rV10MMw7xmh9-5RqB`6z%H5HA7F$v(aQ%HSn5#43M9YWVZZkCgWz)aN=%pO zX@5G|F@mf0w@_0;MOaqL;CQH#@I7Ts`Gkz)H8_+c$uFs(Q|Nai?$625?LOz9g7Xk@ z2IPAEZQ0(~O+u#M1ULIM^M@+Sp}wQ}w}C(0T{u10Lyhut)ng~txP>(|K6)CD<$vcI zX7COhLG9;+G_rq->MeZ@jYf@F~7j}cZn$iTJCs(`|>n<)P%T6|6<%kwsl(b^U`}9JllZwk!ft9rqbLtD?3f! z@yz=xE72+XW~1!~R9$lRU|6ZXL&tSYK%p~2}P)S+gMw-*HfPS;GSB zrP3l_rOCmD?LjSNIX6H_Ht58V?*#1@3WU)xv-)I%^2f!cwG)nyDpSI6DDlamni!Ro z;Pjdmky~DUSdP;3rIiP($dzh7pja>ZYYO*pyD78^5?HD)BBS(!a%IPY zoyQf%p4t@@lH~te{3C!Pcm9wNGWUub_dhPLFIgp#EBZciQ{3PZ=S|Kp6M`0OiIEd! zyr?E9@|4am$t*j{@;Mbkgwbpv9Z$Aum0gz-b?_8vSx2Bf^4rAxrbH~R-TzpOtuL)S zTrSBn!DjK(G6AEw;w{3IVZrF5C(9d4r%=S$N;6n$v|Oiw>+Y~fTuMl!>!e5l7kF+! z#s^WPAY%rLIIG){x0mrizq%)jRRSq(9Ibo>R6jtY3{Xy$)icPavO1n>MvokLUc(s9 zd`5?BM}|asnE}U7IU8kh$k$|kh5QNm*(9I^Eq)^J117#X>GHG4wkWfrd@Boj{+Hl8 zr>@N15DA(8%S*_A2B^GoGB6WoYE63Y=22s8G`;!ln(LHnO?eL1#8Jg}o@x)ciT-?4 ze~`sP8jFMk@MSD2ruyg-IDdSNMCQ29e?qv^Z{VJIX~f`(LG)r3-oTDu2TpY~G4{qJ z-Ee+LGgtdFM<)^$R->}X7FFaa9v1!ZQcjEEH|F_Iv6wZ|U+hR3^5w9G`*7)ad`@qp zb-52c%{s1Ulq0oKt;)LcEvTo1b{R{HnQy_3mZKmqD&)QOEqGC2e8): + count1=count + os.rename(Olddir,Olddir[:5]+'_'+str("{:0>6d}".format(count1))+'.jpg')#重命名 + count+=1 \ No newline at end of file diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/test_net.py" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/test_net.py" new file mode 100644 index 0000000..550b1bf --- /dev/null +++ "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/test_net.py" @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Multi-view test a video classification model.""" + +import numpy as np +import os +import pickle +import torch +from iopath.common.file_io import g_pathmgr + +import slowfast.utils.checkpoint as cu +import slowfast.utils.distributed as du +import slowfast.utils.logging as logging +import slowfast.utils.misc as misc +import slowfast.visualization.tensorboard_vis as tb +from slowfast.datasets import loader +from slowfast.models import build_model +from slowfast.utils.meters import AVAMeter, TestMeter + +logger = logging.get_logger(__name__) + + +@torch.no_grad() +def perform_test(test_loader, model, test_meter, cfg, writer=None): + """ + For classification: + Perform mutli-view testing that uniformly samples N clips from a video along + its temporal axis. For each clip, it takes 3 crops to cover the spatial + dimension, followed by averaging the softmax scores across all Nx3 views to + form a video-level prediction. All video predictions are compared to + ground-truth labels and the final testing performance is logged. + For detection: + Perform fully-convolutional testing on the full frames without crop. + Args: + test_loader (loader): video testing loader. + model (model): the pretrained video model to test. + test_meter (TestMeter): testing meters to log and ensemble the testing + results. + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + writer (TensorboardWriter object, optional): TensorboardWriter object + to writer Tensorboard log. + """ + # Enable eval mode. + model.eval() + test_meter.iter_tic() + + for cur_iter, (inputs, labels, video_idx, meta) in enumerate(test_loader): + if cfg.NUM_GPUS: + # Transfer the data to the current GPU device. + if isinstance(inputs, (list,)): + for i in range(len(inputs)): + inputs[i] = inputs[i].cuda(non_blocking=True) + else: + inputs = inputs.cuda(non_blocking=True) + + # Transfer the data to the current GPU device. + labels = labels.cuda() + video_idx = video_idx.cuda() + for key, val in meta.items(): + if isinstance(val, (list,)): + for i in range(len(val)): + val[i] = val[i].cuda(non_blocking=True) + else: + meta[key] = val.cuda(non_blocking=True) + test_meter.data_toc() + + if cfg.DETECTION.ENABLE: + # Compute the predictions. + preds = model(inputs, meta["boxes"]) + ori_boxes = meta["ori_boxes"] + metadata = meta["metadata"] + + preds = preds.detach().cpu() if cfg.NUM_GPUS else preds.detach() + ori_boxes = ( + ori_boxes.detach().cpu() if cfg.NUM_GPUS else ori_boxes.detach() + ) + metadata = ( + metadata.detach().cpu() if cfg.NUM_GPUS else metadata.detach() + ) + + if cfg.NUM_GPUS > 1: + preds = torch.cat(du.all_gather_unaligned(preds), dim=0) + ori_boxes = torch.cat(du.all_gather_unaligned(ori_boxes), dim=0) + metadata = torch.cat(du.all_gather_unaligned(metadata), dim=0) + + test_meter.iter_toc() + # Update and log stats. + test_meter.update_stats(preds, ori_boxes, metadata) + test_meter.log_iter_stats(None, cur_iter) + else: + # Perform the forward pass. + preds = model(inputs) + + # Gather all the predictions across all the devices to perform ensemble. + if cfg.NUM_GPUS > 1: + preds, labels, video_idx = du.all_gather( + [preds, labels, video_idx] + ) + if cfg.NUM_GPUS: + preds = preds.cpu() + labels = labels.cpu() + video_idx = video_idx.cpu() + + test_meter.iter_toc() + # Update and log stats. + test_meter.update_stats( + preds.detach(), labels.detach(), video_idx.detach() + ) + test_meter.log_iter_stats(cur_iter) + + test_meter.iter_tic() + + # Log epoch stats and print the final testing results. + if not cfg.DETECTION.ENABLE: + all_preds = test_meter.video_preds.clone().detach() + all_labels = test_meter.video_labels + if cfg.NUM_GPUS: + all_preds = all_preds.cpu() + all_labels = all_labels.cpu() + if writer is not None: + writer.plot_eval(preds=all_preds, labels=all_labels) + + if cfg.TEST.SAVE_RESULTS_PATH != "": + save_path = os.path.join(cfg.OUTPUT_DIR, cfg.TEST.SAVE_RESULTS_PATH) + + if du.is_root_proc(): + with g_pathmgr.open(save_path, "wb") as f: + pickle.dump([all_preds, all_labels], f) + + logger.info( + "Successfully saved prediction results to {}".format(save_path) + ) + + test_meter.finalize_metrics() + return test_meter + + +def test(cfg): + """ + Perform multi-view testing on the pretrained video model. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + # Set up environment. + du.init_distributed_training(cfg) + # Set random seed from configs. + np.random.seed(cfg.RNG_SEED) + torch.manual_seed(cfg.RNG_SEED) + + # Setup logging format. + logging.setup_logging(cfg.OUTPUT_DIR) + + # Print config. + logger.info("Test with config:") + logger.info(cfg) + + # Build the video model and print model statistics. + model = build_model(cfg) + if du.is_master_proc() and cfg.LOG_MODEL_INFO: + misc.log_model_info(model, cfg, use_train_input=False) + + cu.load_test_checkpoint(cfg, model) + + # Create video testing loaders. + test_loader = loader.construct_loader(cfg, "test") + logger.info("Testing model for {} iterations".format(len(test_loader))) + + if cfg.DETECTION.ENABLE: + assert cfg.NUM_GPUS == cfg.TEST.BATCH_SIZE or cfg.NUM_GPUS == 0 + test_meter = AVAMeter(len(test_loader), cfg, mode="test") + else: + assert ( + test_loader.dataset.num_videos + % (cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS) + == 0 + ) + # Create meters for multi-view testing. + test_meter = TestMeter( + test_loader.dataset.num_videos + // (cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS), + cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS, + cfg.MODEL.NUM_CLASSES, + len(test_loader), + cfg.DATA.MULTI_LABEL, + cfg.DATA.ENSEMBLE_METHOD, + ) + + # Set up writer for logging to Tensorboard format. + if cfg.TENSORBOARD.ENABLE and du.is_master_proc( + cfg.NUM_GPUS * cfg.NUM_SHARDS + ): + writer = tb.TensorboardWriter(cfg) + else: + writer = None + + # # Perform multi-view test on the entire dataset. + test_meter = perform_test(test_loader, model, test_meter, cfg, writer) + if writer is not None: + writer.close() diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/train_net.py" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/train_net.py" new file mode 100644 index 0000000..24a861c --- /dev/null +++ "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/train_net.py" @@ -0,0 +1,524 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Train a video classification model.""" + +import numpy as np +import pprint +import torch +from fvcore.nn.precise_bn import get_bn_modules, update_bn_stats + +import slowfast.models.losses as losses +import slowfast.models.optimizer as optim +import slowfast.utils.checkpoint as cu +import slowfast.utils.distributed as du +import slowfast.utils.logging as logging +import slowfast.utils.metrics as metrics +import slowfast.utils.misc as misc +import slowfast.visualization.tensorboard_vis as tb +from slowfast.datasets import loader +from slowfast.datasets.mixup import MixUp +from slowfast.models import build_model +from slowfast.utils.meters import AVAMeter, EpochTimer, TrainMeter, ValMeter +from slowfast.utils.multigrid import MultigridSchedule + +logger = logging.get_logger(__name__) + + +def train_epoch( + train_loader, model, optimizer, train_meter, cur_epoch, cfg, writer=None +): + """ + Perform the video training for one epoch. + Args: + train_loader (loader): video training loader. + model (model): the video model to train. + optimizer (optim): the optimizer to perform optimization on the model's + parameters. + train_meter (TrainMeter): training meters to log the training performance. + cur_epoch (int): current epoch of training. + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + writer (TensorboardWriter, optional): TensorboardWriter object + to writer Tensorboard log. + """ + # Enable train mode. + model.train() + train_meter.iter_tic() + data_size = len(train_loader) + + if cfg.MIXUP.ENABLE: + mixup_fn = MixUp( + mixup_alpha=cfg.MIXUP.ALPHA, + cutmix_alpha=cfg.MIXUP.CUTMIX_ALPHA, + mix_prob=cfg.MIXUP.PROB, + switch_prob=cfg.MIXUP.SWITCH_PROB, + label_smoothing=cfg.MIXUP.LABEL_SMOOTH_VALUE, + num_classes=cfg.MODEL.NUM_CLASSES, + ) + + for cur_iter, (inputs, labels, _, meta) in enumerate(train_loader): + # Transfer the data to the current GPU device. + if cfg.NUM_GPUS: + if isinstance(inputs, (list,)): + for i in range(len(inputs)): + inputs[i] = inputs[i].cuda(non_blocking=True) + else: + inputs = inputs.cuda(non_blocking=True) + labels = labels.cuda() + for key, val in meta.items(): + if isinstance(val, (list,)): + for i in range(len(val)): + val[i] = val[i].cuda(non_blocking=True) + else: + meta[key] = val.cuda(non_blocking=True) + + # Update the learning rate. + lr = optim.get_epoch_lr(cur_epoch + float(cur_iter) / data_size, cfg) + optim.set_lr(optimizer, lr) + + train_meter.data_toc() + if cfg.MIXUP.ENABLE: + samples, labels = mixup_fn(inputs[0], labels) + inputs[0] = samples + + if cfg.DETECTION.ENABLE: + preds = model(inputs, meta["boxes"]) + else: + preds = model(inputs) + # Explicitly declare reduction to mean. + loss_fun = losses.get_loss_func(cfg.MODEL.LOSS_FUNC)(reduction="mean") + + # Compute the loss. + loss = loss_fun(preds, labels) + + # check Nan Loss. + misc.check_nan_losses(loss) + + # Perform the backward pass. + optimizer.zero_grad() + loss.backward() + # Update the parameters. + optimizer.step() + + if cfg.MIXUP.ENABLE: + _top_max_k_vals, top_max_k_inds = torch.topk( + labels, 2, dim=1, largest=True, sorted=True + ) + idx_top1 = torch.arange(labels.shape[0]), top_max_k_inds[:, 0] + idx_top2 = torch.arange(labels.shape[0]), top_max_k_inds[:, 1] + preds = preds.detach() + preds[idx_top1] += preds[idx_top2] + preds[idx_top2] = 0.0 + labels = top_max_k_inds[:, 0] + + if cfg.DETECTION.ENABLE: + if cfg.NUM_GPUS > 1: + loss = du.all_reduce([loss])[0] + loss = loss.item() + + # Update and log stats. + train_meter.update_stats(None, None, None, loss, lr) + # write to tensorboard format if available. + if writer is not None: + writer.add_scalars( + {"Train/loss": loss, "Train/lr": lr}, + global_step=data_size * cur_epoch + cur_iter, + ) + + else: + top1_err, top5_err = None, None + if cfg.DATA.MULTI_LABEL: + # Gather all the predictions across all the devices. + if cfg.NUM_GPUS > 1: + [loss] = du.all_reduce([loss]) + loss = loss.item() + else: + # Compute the errors. + num_topks_correct = metrics.topks_correct(preds, labels, (1, 5)) + top1_err, top5_err = [ + (1.0 - x / preds.size(0)) * 100.0 for x in num_topks_correct + ] + # Gather all the predictions across all the devices. + if cfg.NUM_GPUS > 1: + loss, top1_err, top5_err = du.all_reduce( + [loss, top1_err, top5_err] + ) + + # Copy the stats from GPU to CPU (sync point). + loss, top1_err, top5_err = ( + loss.item(), + top1_err.item(), + top5_err.item(), + ) + + # Update and log stats. + train_meter.update_stats( + top1_err, + top5_err, + loss, + lr, + inputs[0].size(0) + * max( + cfg.NUM_GPUS, 1 + ), # If running on CPU (cfg.NUM_GPUS == 1), use 1 to represent 1 CPU. + ) + # write to tensorboard format if available. + if writer is not None: + writer.add_scalars( + { + "Train/loss": loss, + "Train/lr": lr, + "Train/Top1_err": top1_err, + "Train/Top5_err": top5_err, + }, + global_step=data_size * cur_epoch + cur_iter, + ) + + train_meter.iter_toc() # measure allreduce for this meter + train_meter.log_iter_stats(cur_epoch, cur_iter) + train_meter.iter_tic() + + # Log epoch stats. + train_meter.log_epoch_stats(cur_epoch) + train_meter.reset() + + +@torch.no_grad() +def eval_epoch(val_loader, model, val_meter, cur_epoch, cfg, writer=None): + """ + Evaluate the model on the val set. + Args: + val_loader (loader): data loader to provide validation data. + model (model): model to evaluate the performance. + val_meter (ValMeter): meter instance to record and calculate the metrics. + cur_epoch (int): number of the current epoch of training. + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + writer (TensorboardWriter, optional): TensorboardWriter object + to writer Tensorboard log. + """ + + # Evaluation mode enabled. The running stats would not be updated. + model.eval() + val_meter.iter_tic() + + for cur_iter, (inputs, labels, _, meta) in enumerate(val_loader): + if cfg.NUM_GPUS: + # Transferthe data to the current GPU device. + if isinstance(inputs, (list,)): + for i in range(len(inputs)): + inputs[i] = inputs[i].cuda(non_blocking=True) + else: + inputs = inputs.cuda(non_blocking=True) + labels = labels.cuda() + for key, val in meta.items(): + if isinstance(val, (list,)): + for i in range(len(val)): + val[i] = val[i].cuda(non_blocking=True) + else: + meta[key] = val.cuda(non_blocking=True) + val_meter.data_toc() + + if cfg.DETECTION.ENABLE: + # Compute the predictions. + preds = model(inputs, meta["boxes"]) + ori_boxes = meta["ori_boxes"] + metadata = meta["metadata"] + + if cfg.NUM_GPUS: + preds = preds.cpu() + ori_boxes = ori_boxes.cpu() + metadata = metadata.cpu() + + if cfg.NUM_GPUS > 1: + preds = torch.cat(du.all_gather_unaligned(preds), dim=0) + ori_boxes = torch.cat(du.all_gather_unaligned(ori_boxes), dim=0) + metadata = torch.cat(du.all_gather_unaligned(metadata), dim=0) + + val_meter.iter_toc() + # Update and log stats. + val_meter.update_stats(preds, ori_boxes, metadata) + + else: + preds = model(inputs) + + if cfg.DATA.MULTI_LABEL: + if cfg.NUM_GPUS > 1: + preds, labels = du.all_gather([preds, labels]) + else: + # Compute the errors. + num_topks_correct = metrics.topks_correct(preds, labels, (1, 5)) + + # Combine the errors across the GPUs. + top1_err, top5_err = [ + (1.0 - x / preds.size(0)) * 100.0 for x in num_topks_correct + ] + if cfg.NUM_GPUS > 1: + top1_err, top5_err = du.all_reduce([top1_err, top5_err]) + + # Copy the errors from GPU to CPU (sync point). + top1_err, top5_err = top1_err.item(), top5_err.item() + + val_meter.iter_toc() + # Update and log stats. + val_meter.update_stats( + top1_err, + top5_err, + inputs[0].size(0) + * max( + cfg.NUM_GPUS, 1 + ), # If running on CPU (cfg.NUM_GPUS == 1), use 1 to represent 1 CPU. + ) + # write to tensorboard format if available. + if writer is not None: + writer.add_scalars( + {"Val/Top1_err": top1_err, "Val/Top5_err": top5_err}, + global_step=len(val_loader) * cur_epoch + cur_iter, + ) + + val_meter.update_predictions(preds, labels) + + val_meter.log_iter_stats(cur_epoch, cur_iter) + val_meter.iter_tic() + + # Log epoch stats. + val_meter.log_epoch_stats(cur_epoch) + # write to tensorboard format if available. + if writer is not None: + if cfg.DETECTION.ENABLE: + writer.add_scalars( + {"Val/mAP": val_meter.full_map}, global_step=cur_epoch + ) + else: + all_preds = [pred.clone().detach() for pred in val_meter.all_preds] + all_labels = [ + label.clone().detach() for label in val_meter.all_labels + ] + if cfg.NUM_GPUS: + all_preds = [pred.cpu() for pred in all_preds] + all_labels = [label.cpu() for label in all_labels] + writer.plot_eval( + preds=all_preds, labels=all_labels, global_step=cur_epoch + ) + + val_meter.reset() + + +def calculate_and_update_precise_bn(loader, model, num_iters=200, use_gpu=True): + """ + Update the stats in bn layers by calculate the precise stats. + Args: + loader (loader): data loader to provide training data. + model (model): model to update the bn stats. + num_iters (int): number of iterations to compute and update the bn stats. + use_gpu (bool): whether to use GPU or not. + """ + + def _gen_loader(): + for inputs, *_ in loader: + if use_gpu: + if isinstance(inputs, (list,)): + for i in range(len(inputs)): + inputs[i] = inputs[i].cuda(non_blocking=True) + else: + inputs = inputs.cuda(non_blocking=True) + yield inputs + + # Update the bn stats. + update_bn_stats(model, _gen_loader(), num_iters) + + +def build_trainer(cfg): + """ + Build training model and its associated tools, including optimizer, + dataloaders and meters. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + Returns: + model (nn.Module): training model. + optimizer (Optimizer): optimizer. + train_loader (DataLoader): training data loader. + val_loader (DataLoader): validatoin data loader. + precise_bn_loader (DataLoader): training data loader for computing + precise BN. + train_meter (TrainMeter): tool for measuring training stats. + val_meter (ValMeter): tool for measuring validation stats. + """ + # Build the video model and print model statistics. + model = build_model(cfg) + if du.is_master_proc() and cfg.LOG_MODEL_INFO: + misc.log_model_info(model, cfg, use_train_input=True) + + # Construct the optimizer. + optimizer = optim.construct_optimizer(model, cfg) + + # Create the video train and val loaders. + train_loader = loader.construct_loader(cfg, "train") + val_loader = loader.construct_loader(cfg, "val") + precise_bn_loader = loader.construct_loader( + cfg, "train", is_precise_bn=True + ) + # Create meters. + train_meter = TrainMeter(len(train_loader), cfg) + val_meter = ValMeter(len(val_loader), cfg) + + return ( + model, + optimizer, + train_loader, + val_loader, + precise_bn_loader, + train_meter, + val_meter, + ) + + +def train(cfg): + """ + Train a video model for many epochs on train set and evaluate it on val set. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + # Set up environment. + du.init_distributed_training(cfg) + # Set random seed from configs. + np.random.seed(cfg.RNG_SEED) + torch.manual_seed(cfg.RNG_SEED) + + # Setup logging format. + logging.setup_logging(cfg.OUTPUT_DIR) + + # Init multigrid. + multigrid = None + if cfg.MULTIGRID.LONG_CYCLE or cfg.MULTIGRID.SHORT_CYCLE: + multigrid = MultigridSchedule() + cfg = multigrid.init_multigrid(cfg) + if cfg.MULTIGRID.LONG_CYCLE: + cfg, _ = multigrid.update_long_cycle(cfg, cur_epoch=0) + # Print config. + logger.info("Train with config:") + logger.info(pprint.pformat(cfg)) + + # Build the video model and print model statistics. + model = build_model(cfg) + if du.is_master_proc() and cfg.LOG_MODEL_INFO: + misc.log_model_info(model, cfg, use_train_input=True) + + # Construct the optimizer. + optimizer = optim.construct_optimizer(model, cfg) + + # Load a checkpoint to resume training if applicable. + start_epoch = cu.load_train_checkpoint(cfg, model, optimizer) + + # Create the video train and val loaders. + train_loader = loader.construct_loader(cfg, "train") + val_loader = loader.construct_loader(cfg, "val") + precise_bn_loader = ( + loader.construct_loader(cfg, "train", is_precise_bn=True) + if cfg.BN.USE_PRECISE_STATS + else None + ) + + # Create meters. + if cfg.DETECTION.ENABLE: + train_meter = AVAMeter(len(train_loader), cfg, mode="train") + val_meter = AVAMeter(len(val_loader), cfg, mode="val") + else: + train_meter = TrainMeter(len(train_loader), cfg) + val_meter = ValMeter(len(val_loader), cfg) + + # set up writer for logging to Tensorboard format. + if cfg.TENSORBOARD.ENABLE and du.is_master_proc( + cfg.NUM_GPUS * cfg.NUM_SHARDS + ): + writer = tb.TensorboardWriter(cfg) + else: + writer = None + + # Perform the training loop. + logger.info("Start epoch: {}".format(start_epoch + 1)) + + epoch_timer = EpochTimer() + + for cur_epoch in range(start_epoch, cfg.SOLVER.MAX_EPOCH): + if cfg.MULTIGRID.LONG_CYCLE: + cfg, changed = multigrid.update_long_cycle(cfg, cur_epoch) + if changed: + ( + model, + optimizer, + train_loader, + val_loader, + precise_bn_loader, + train_meter, + val_meter, + ) = build_trainer(cfg) + + # Load checkpoint. + if cu.has_checkpoint(cfg.OUTPUT_DIR): + last_checkpoint = cu.get_last_checkpoint(cfg.OUTPUT_DIR) + assert "{:05d}.pyth".format(cur_epoch) in last_checkpoint + else: + last_checkpoint = cfg.TRAIN.CHECKPOINT_FILE_PATH + logger.info("Load from {}".format(last_checkpoint)) + cu.load_checkpoint( + last_checkpoint, model, cfg.NUM_GPUS > 1, optimizer + ) + + # Shuffle the dataset. + loader.shuffle_dataset(train_loader, cur_epoch) + + # Train for one epoch. + epoch_timer.epoch_tic() + train_epoch( + train_loader, model, optimizer, train_meter, cur_epoch, cfg, writer + ) + epoch_timer.epoch_toc() + logger.info( + f"Epoch {cur_epoch} takes {epoch_timer.last_epoch_time():.2f}s. Epochs " + f"from {start_epoch} to {cur_epoch} take " + f"{epoch_timer.avg_epoch_time():.2f}s in average and " + f"{epoch_timer.median_epoch_time():.2f}s in median." + ) + logger.info( + f"For epoch {cur_epoch}, each iteraction takes " + f"{epoch_timer.last_epoch_time()/len(train_loader):.2f}s in average. " + f"From epoch {start_epoch} to {cur_epoch}, each iteraction takes " + f"{epoch_timer.avg_epoch_time()/len(train_loader):.2f}s in average." + ) + + is_checkp_epoch = cu.is_checkpoint_epoch( + cfg, + cur_epoch, + None if multigrid is None else multigrid.schedule, + ) + is_eval_epoch = misc.is_eval_epoch( + cfg, cur_epoch, None if multigrid is None else multigrid.schedule + ) + + # Compute precise BN stats. + if ( + (is_checkp_epoch or is_eval_epoch) + and cfg.BN.USE_PRECISE_STATS + and len(get_bn_modules(model)) > 0 + ): + calculate_and_update_precise_bn( + precise_bn_loader, + model, + min(cfg.BN.NUM_BATCHES_PRECISE, len(precise_bn_loader)), + cfg.NUM_GPUS > 0, + ) + _ = misc.aggregate_sub_bn_stats(model) + + # Save a checkpoint. + if is_checkp_epoch: + cu.save_checkpoint(cfg.OUTPUT_DIR, model, optimizer, cur_epoch, cfg) + # Evaluate the model on validation set. + if is_eval_epoch: + eval_epoch(val_loader, model, val_meter, cur_epoch, cfg, writer) + + if writer is not None: + writer.close() diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/visualization.py" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/visualization.py" new file mode 100644 index 0000000..905863b --- /dev/null +++ "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/visualization.py" @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import numpy as np +import pickle +import torch +import tqdm +from iopath.common.file_io import g_pathmgr + +import slowfast.datasets.utils as data_utils +import slowfast.utils.checkpoint as cu +import slowfast.utils.distributed as du +import slowfast.utils.logging as logging +import slowfast.utils.misc as misc +import slowfast.visualization.tensorboard_vis as tb +from slowfast.datasets import loader +from slowfast.models import build_model +from slowfast.visualization.gradcam_utils import GradCAM +from slowfast.visualization.prediction_vis import WrongPredictionVis +from slowfast.visualization.utils import ( + GetWeightAndActivation, + process_layer_index_data, +) +from slowfast.visualization.video_visualizer import VideoVisualizer + +logger = logging.get_logger(__name__) + + +def run_visualization(vis_loader, model, cfg, writer=None): + """ + Run model visualization (weights, activations and model inputs) and visualize + them on Tensorboard. + Args: + vis_loader (loader): video visualization loader. + model (model): the video model to visualize. + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + writer (TensorboardWriter, optional): TensorboardWriter object + to writer Tensorboard log. + """ + n_devices = cfg.NUM_GPUS * cfg.NUM_SHARDS + prefix = "module/" if n_devices > 1 else "" + # Get a list of selected layer names and indexing. + layer_ls, indexing_dict = process_layer_index_data( + cfg.TENSORBOARD.MODEL_VIS.LAYER_LIST, layer_name_prefix=prefix + ) + logger.info("Start Model Visualization.") + # Register hooks for activations. + model_vis = GetWeightAndActivation(model, layer_ls) + + if writer is not None and cfg.TENSORBOARD.MODEL_VIS.MODEL_WEIGHTS: + layer_weights = model_vis.get_weights() + writer.plot_weights_and_activations( + layer_weights, tag="Layer Weights/", heat_map=False + ) + + video_vis = VideoVisualizer( + cfg.MODEL.NUM_CLASSES, + cfg.TENSORBOARD.CLASS_NAMES_PATH, + cfg.TENSORBOARD.MODEL_VIS.TOPK_PREDS, + cfg.TENSORBOARD.MODEL_VIS.COLORMAP, + ) + if n_devices > 1: + grad_cam_layer_ls = [ + "module/" + layer + for layer in cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST + ] + else: + grad_cam_layer_ls = cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST + + if cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE: + gradcam = GradCAM( + model, + target_layers=grad_cam_layer_ls, + data_mean=cfg.DATA.MEAN, + data_std=cfg.DATA.STD, + colormap=cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.COLORMAP, + ) + logger.info("Finish drawing weights.") + global_idx = -1 + for inputs, labels, _, meta in tqdm.tqdm(vis_loader): + if cfg.NUM_GPUS: + # Transfer the data to the current GPU device. + if isinstance(inputs, (list,)): + for i in range(len(inputs)): + inputs[i] = inputs[i].cuda(non_blocking=True) + else: + inputs = inputs.cuda(non_blocking=True) + labels = labels.cuda() + for key, val in meta.items(): + if isinstance(val, (list,)): + for i in range(len(val)): + val[i] = val[i].cuda(non_blocking=True) + else: + meta[key] = val.cuda(non_blocking=True) + + if cfg.DETECTION.ENABLE: + activations, preds = model_vis.get_activations( + inputs, meta["boxes"] + ) + else: + activations, preds = model_vis.get_activations(inputs) + if cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE: + if cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.USE_TRUE_LABEL: + inputs, preds = gradcam(inputs, labels=labels) + else: + inputs, preds = gradcam(inputs) + if cfg.NUM_GPUS: + inputs = du.all_gather_unaligned(inputs) + activations = du.all_gather_unaligned(activations) + preds = du.all_gather_unaligned(preds) + if isinstance(inputs[0], list): + for i in range(len(inputs)): + for j in range(len(inputs[0])): + inputs[i][j] = inputs[i][j].cpu() + else: + inputs = [inp.cpu() for inp in inputs] + preds = [pred.cpu() for pred in preds] + else: + inputs, activations, preds = [inputs], [activations], [preds] + + boxes = [None] * max(n_devices, 1) + if cfg.DETECTION.ENABLE and cfg.NUM_GPUS: + boxes = du.all_gather_unaligned(meta["boxes"]) + boxes = [box.cpu() for box in boxes] + + if writer is not None: + total_vids = 0 + for i in range(max(n_devices, 1)): + cur_input = inputs[i] + cur_activations = activations[i] + cur_batch_size = cur_input[0].shape[0] + cur_preds = preds[i] + cur_boxes = boxes[i] + for cur_batch_idx in range(cur_batch_size): + global_idx += 1 + total_vids += 1 + if ( + cfg.TENSORBOARD.MODEL_VIS.INPUT_VIDEO + or cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE + ): + for path_idx, input_pathway in enumerate(cur_input): + if cfg.TEST.DATASET == "ava" and cfg.AVA.BGR: + video = input_pathway[ + cur_batch_idx, [2, 1, 0], ... + ] + else: + video = input_pathway[cur_batch_idx] + + if not cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE: + # Permute to (T, H, W, C) from (C, T, H, W). + video = video.permute(1, 2, 3, 0) + video = data_utils.revert_tensor_normalize( + video, cfg.DATA.MEAN, cfg.DATA.STD + ) + else: + # Permute from (T, C, H, W) to (T, H, W, C) + video = video.permute(0, 2, 3, 1) + bboxes = ( + None if cur_boxes is None else cur_boxes[:, 1:] + ) + cur_prediction = ( + cur_preds + if cfg.DETECTION.ENABLE + else cur_preds[cur_batch_idx] + ) + video = video_vis.draw_clip( + video, cur_prediction, bboxes=bboxes + ) + video = ( + torch.from_numpy(np.array(video)) + .permute(0, 3, 1, 2) + .unsqueeze(0) + ) + writer.add_video( + video, + tag="Input {}/Pathway {}".format( + global_idx, path_idx + 1 + ), + ) + if cfg.TENSORBOARD.MODEL_VIS.ACTIVATIONS: + writer.plot_weights_and_activations( + cur_activations, + tag="Input {}/Activations: ".format(global_idx), + batch_idx=cur_batch_idx, + indexing_dict=indexing_dict, + ) + + +def perform_wrong_prediction_vis(vis_loader, model, cfg): + """ + Visualize video inputs with wrong predictions on Tensorboard. + Args: + vis_loader (loader): video visualization loader. + model (model): the video model to visualize. + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + wrong_prediction_visualizer = WrongPredictionVis(cfg=cfg) + for batch_idx, (inputs, labels, _, _) in tqdm.tqdm(enumerate(vis_loader)): + if cfg.NUM_GPUS: + # Transfer the data to the current GPU device. + if isinstance(inputs, (list,)): + for i in range(len(inputs)): + inputs[i] = inputs[i].cuda(non_blocking=True) + else: + inputs = inputs.cuda(non_blocking=True) + labels = labels.cuda() + + # Some model modify the original input. + inputs_clone = [inp.clone() for inp in inputs] + + preds = model(inputs) + + if cfg.NUM_GPUS > 1: + preds, labels = du.all_gather([preds, labels]) + if isinstance(inputs_clone, (list,)): + inputs_clone = du.all_gather(inputs_clone) + else: + inputs_clone = du.all_gather([inputs_clone])[0] + + if cfg.NUM_GPUS: + # Transfer the data to the current CPU device. + labels = labels.cpu() + preds = preds.cpu() + if isinstance(inputs_clone, (list,)): + for i in range(len(inputs_clone)): + inputs_clone[i] = inputs_clone[i].cpu() + else: + inputs_clone = inputs_clone.cpu() + + # If using CPU (NUM_GPUS = 0), 1 represent 1 CPU. + n_devices = max(cfg.NUM_GPUS, 1) + for device_idx in range(1, n_devices + 1): + wrong_prediction_visualizer.visualize_vid( + video_input=inputs_clone, + labels=labels, + preds=preds.detach().clone(), + batch_idx=device_idx * batch_idx, + ) + + logger.info( + "Class indices with wrong predictions: {}".format( + sorted(wrong_prediction_visualizer.wrong_class_prediction) + ) + ) + wrong_prediction_visualizer.clean() + + +def visualize(cfg): + """ + Perform layer weights and activations visualization on the model. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + if cfg.TENSORBOARD.ENABLE and ( + cfg.TENSORBOARD.MODEL_VIS.ENABLE + or cfg.TENSORBOARD.WRONG_PRED_VIS.ENABLE + ): + # Set up environment. + du.init_distributed_training(cfg) + # Set random seed from configs. + np.random.seed(cfg.RNG_SEED) + torch.manual_seed(cfg.RNG_SEED) + + # Setup logging format. + logging.setup_logging(cfg.OUTPUT_DIR) + + # Print config. + logger.info("Model Visualization with config:") + logger.info(cfg) + + # Build the video model and print model statistics. + model = build_model(cfg) + model.eval() + if du.is_master_proc() and cfg.LOG_MODEL_INFO: + misc.log_model_info(model, cfg, use_train_input=False) + + cu.load_test_checkpoint(cfg, model) + + # Create video testing loaders. + vis_loader = loader.construct_loader(cfg, "test") + + if cfg.DETECTION.ENABLE: + assert cfg.NUM_GPUS == cfg.TEST.BATCH_SIZE or cfg.NUM_GPUS == 0 + + # Set up writer for logging to Tensorboard format. + if du.is_master_proc(cfg.NUM_GPUS * cfg.NUM_SHARDS): + writer = tb.TensorboardWriter(cfg) + else: + writer = None + if cfg.TENSORBOARD.PREDICTIONS_PATH != "": + assert not cfg.DETECTION.ENABLE, "Detection is not supported." + logger.info( + "Visualizing class-level performance from saved results..." + ) + if writer is not None: + with g_pathmgr.open( + cfg.TENSORBOARD.PREDICTIONS_PATH, "rb" + ) as f: + preds, labels = pickle.load(f, encoding="latin1") + + writer.plot_eval(preds, labels) + + if cfg.TENSORBOARD.MODEL_VIS.ENABLE: + if cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE: + assert ( + not cfg.DETECTION.ENABLE + ), "Detection task is currently not supported for Grad-CAM visualization." + if cfg.MODEL.ARCH in cfg.MODEL.SINGLE_PATHWAY_ARCH: + assert ( + len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST) == 1 + ), "The number of chosen CNN layers must be equal to the number of pathway(s), given {} layer(s).".format( + len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST) + ) + elif cfg.MODEL.ARCH in cfg.MODEL.MULTI_PATHWAY_ARCH: + assert ( + len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST) == 2 + ), "The number of chosen CNN layers must be equal to the number of pathway(s), given {} layer(s).".format( + len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST) + ) + else: + raise NotImplementedError( + "Model arch {} is not in {}".format( + cfg.MODEL.ARCH, + cfg.MODEL.SINGLE_PATHWAY_ARCH + + cfg.MODEL.MULTI_PATHWAY_ARCH, + ) + ) + logger.info( + "Visualize model analysis for {} iterations".format( + len(vis_loader) + ) + ) + # Run visualization on the model + run_visualization(vis_loader, model, cfg, writer) + if cfg.TENSORBOARD.WRONG_PRED_VIS.ENABLE: + logger.info( + "Visualize Wrong Predictions for {} iterations".format( + len(vis_loader) + ) + ) + perform_wrong_prediction_vis(vis_loader, model, cfg) + + if writer is not None: + writer.close() -- Gitee From e04d68c950e5e64978c3f44db6bd8e8da17800a4 Mon Sep 17 00:00:00 2001 From: Aurora_zxy <9166856+zombie233@user.noreply.gitee.com> Date: Sat, 10 Jul 2021 12:27:21 +0000 Subject: [PATCH 04/10] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20slowfast?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../slowfast/.keep" | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/.keep" diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/.keep" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/.keep" new file mode 100644 index 0000000..e69de29 -- Gitee From b1fd9ffc8110f0841e002ebe72067f9c9fd4b28b Mon Sep 17 00:00:00 2001 From: Aurora_zxy <9166856+zombie233@user.noreply.gitee.com> Date: Sat, 10 Jul 2021 12:30:07 +0000 Subject: [PATCH 05/10] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6=20co?= =?UTF-8?q?de/2019302110407-=E5=BC=A0=E7=BF=94=E5=AE=87-=E5=8A=A8=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.keep" | 0 .../slowfast/.keep" | 0 .../tools/.keep" | 0 .../__pycache__/demo_net.cpython-37.pyc" | Bin 3087 -> 0 bytes .../__pycache__/test_net.cpython-37.pyc" | Bin 4780 -> 0 bytes .../__pycache__/train_net.cpython-37.pyc" | Bin 11571 -> 0 bytes .../__pycache__/visualization.cpython-37.pyc" | Bin 8318 -> 0 bytes .../tools/benchmark.py" | 25 - .../tools/demo_net.py" | 119 ---- .../tools/run_net.py" | 49 -- .../tools/ss.py" | 19 - .../tools/test_net.py" | 202 ------- .../tools/train_net.py" | 524 ------------------ .../tools/visualization.py" | 347 ------------ 14 files changed, 1285 deletions(-) delete mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/.keep" delete mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/.keep" delete mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/.keep" delete mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/demo_net.cpython-37.pyc" delete mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/test_net.cpython-37.pyc" delete mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/train_net.cpython-37.pyc" delete mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/visualization.cpython-37.pyc" delete mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/benchmark.py" delete mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/demo_net.py" delete mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/run_net.py" delete mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/ss.py" delete mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/test_net.py" delete mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/train_net.py" delete mode 100644 "code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/visualization.py" diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/.keep" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/.keep" deleted file mode 100644 index e69de29..0000000 diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/.keep" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/.keep" deleted file mode 100644 index e69de29..0000000 diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/.keep" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/.keep" deleted file mode 100644 index e69de29..0000000 diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/demo_net.cpython-37.pyc" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/demo_net.cpython-37.pyc" deleted file mode 100644 index 4fca089fd42266b3d0389b24d012d1de31e24bdb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3087 zcmcguOK&5`5uO*x;X{;Gt6eK{98WfD?-EF)_>zZgIH5#pEm$H6l9JW~2cywUlN@TM zhuhO!JxpDqTpjEo$T6Uf34$DQ2=W{J6Z&T45Nv*e;e)GsBTRgRji$x*j;IU2SBqZVeOtesU+ zRu6NLX`4~r&da$G7NVkElyoL6MRWFCRJO}hU0^u-6Yf z9y4S~kypAs4Q~Y>8Wo!dE%~>qC|nt;Q*ELQ^($3XxG~Ml=5lS)9pIJ`StlCyiVzdyk{vS`WzPX|t ztI7ZP13!PM{vK|5k(;Q~pIj-ba>^%aP5E8;xprY*+xo<--vxuS+~5H_JV%0YSjpLgD|l?ms&@}>cwN~ zTLGQz!@WrupY+^>ucf=zeA08rAx~CEXEV7Tb0gx6So|CUmRXMjPMC`!xo)-2_#md% z4aedGms^bRF{2iDlVdAB8WN8u>4MAP7C{&if2JGnZ~{4EQR0y#aoHK%uotsPzSc^{ z-oSDb>rmbyIsB=082E>l3lBM@pl5i;4CUHhCAJ`clkDOzX;gH>lT?EU* z)zZl#06-MebaT2#m|h&>Lr21h&~vJtr_M>tjtNU%;*o$9I`~%PT?85jZvGN8{`|uY z`D!ZxfI;Wdn4^FDfGOY^G&QvbZrHQpo|TYiV?sS*g@pF`V733n(m$Sjeg3E4R4PKF zBavlb%QzB7LWnQScH@cDsn=`ph?sc;QH)#)Nao-w!^6c~0^S`t*Dxpw&Ao1Guj|yd z+ailhfHDI;D0Dbc7`6ItQ_SyFH|jgi*7i=_X;r(Qh{et3?ryW;Z0=M$9j8&x{)rWD@Q&E({(-1STON zBcVfO5PF|*QP`$FIhB%?0-ML<%D0oxF25c~5M_DH$rxjUEuee`Ln#zF@Vf&JuuTkE z%tX879)-l=v4f=tZUDT$Dr_A#ee-B-5JzOK0};CgSn#t5I>Od?9ES-YLKHica6klM zvN3g#aSyg7pF^h<%W6q0X$7^QFRO;Gsag1$ve#fN=?kz{28jiI5x9nCs%0286?Q;x z!Yay{Fk3)Pug%y;px?txz%K(W3vClx@@MFvBtV&C4`I}Xy01dHddQ);T&fr9M7;!< zrj%!<8Gv*wOWH8YKITyTU|pvsH&g2L0nbnKya1&~ztpaC@uGkfq}30jx|* zSYTkB!;zP#^S(A+NVQM2Whk8}D&YxQn@v%B4Fh-|%4-PozKcfjbP zC=<@vXg-CUMW3(_kUfXlRjoPKqUb1=!v%vs^CRg=k@JIO6uM`^7{&n=880L*Wh*H9 zHi~9b+^=M0=-YPzxo`eAIY;rS4Dz3&^e?dcHFg-jY4;&e!ob)(1S(mEPBE7-vLNCh zx-?U}4{?Yzdje~fWs!m07@e_2n8`=5`$%m-XP0m4;_8?~36;=izdvK%Dk7^XJYUzD zm`R+Fm|}nMrX9~+nMP1lZ#|JD(xmo-H|&$S7{)G?VKy_dJuBj?x9|1pMhIW_qc

zA;|S#))a{g07w$$0nklHmMq6forP&BZTOn`r6Z&d%#Q$q#v%C_QC0c@Z4N)kZOCvP L8svFFH$VLj7BX?m diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/test_net.cpython-37.pyc" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/test_net.cpython-37.pyc" deleted file mode 100644 index 69cf893a15dc452969671bca21d9a3615ca926f6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4780 zcmcInOK%(36`mL06!owyTZ(K?oW!vbTW*^+1&p9DZPB&>S#m`>E(Q*!BhHW>Y97p; zOIdb@Kv88`w?=+IL2d$c(M{1+|3ZN-y6EC=3$)0lKOn1q=MJfdk!>l=eVlvldEN8U z-IQe4&50@c#wtXsOA4a<;o&dOodgM3)9 z3TlR13`?4#Dq-2GOd!<>WX7sZAhQ$5oHaLr%ugT-*1`mGE?ks!&Ra_oVgwh$i`GTC z&IQZiinSuw`QV*!)mjZNS(jABn;TvBu8dX7^m3zj$BJ8V3vTf%^+>g@x+S;#m14c; zYE5OW`s(JRLBRc+2fp{hG7HG?5J4ZC|Tf|qwcMo5(q$^6Q zj+8NMmTDuF<~Q2m2G>&MNZr8ti%*|v!`w4tm>(8~#bGJcdpi3!H_|-Mb<`t`1&5#V zJTIjABj~xId~tDDPV;GbS?Lw&m0qi~`%L+A`>7ID)BF@V%%_D>ajXmrX;IF_!)Iw} zR2nOw@fbWS!z#E`CtUt#YBW*Z?m9eZ0YnXGGXGu?BeU#3OX2!~>Hddm1wgzr?c0aA1I+JVM z&(8MJ*;8k7ozZmj`v&O#na}p-`21)gof)g-C9fgFF|2kBs~y8;k72XW<=o`O7Soxu zmd=iK`F`i!QkolUsuCTLc1LO^tK_LM1^kwP4~uv8vHDCO&d{4mi8W?IJ7|d;1v%5Z z(7t$<8)*d$8jCbeIfJh%WspAfjKT@^F5_^>^u<u+>c=9hYxM^^wdGx%5=EksZPNZMU?1~tz zIo}{S$|ipIWA;{D-j?vk9_z#`G{XT8PQ|hxbxr=<;pQOn={$I8CQjH7yu{oF9lxKL z9TtbjA@2ln)RhE%o*)~9{g^p{=^XmWy5#LS?dS614b$f)clJT)6SK`?P~u1#@qx#{ z(K8by2j2-KE!PjdC_xUnVRqsmh+lZFx%bj^fa`QK5pf~z@X$FllXlEN)B()|QxKTj zho2xLc`t~Aq$f3*|_BA?#Ab5GR z9Y+UoFd*ShQA9?SY=D9{a6;((!spN90hi{?XxCXc$w<=UlQ4FcY0c}CjC*^c>QvAS zoZY57G7+2CvwVD8vV>JQbNmQ)$jp@C;Q5Hi$yLcNq~vX0Z%vc;8fZ*xB9G*=!yM9X zcx5l#3p~kYf+lT5lF$z?i(-?U@Mw3s=Jk7>?ly`GNW)YezYAa6@VMjSHQG*O?%^@$ zolHMELo9(Ob)1CX%5L3qy^bTP_g^04y$4pWL=8(pl}yI=MD7l_#nhEHjl=>IX5m zNp)eg8qJoNZ`MC=*t?D9ldV?Meq3)o6k41JgBllM^kP2}<(((3$4^@J#^$c5`iaeA zkm$3x%``lVFycOJk?;HM{Qy>S2VtL@9YjA9h97lek*93raBEH`4F8p9<8_(ePSzGh zMTSrok|HO=Oq3-|#ttmj>56>jPFAr!U{ZIH_oM!RC#*&ilrj~4_fQz*H$wG=zVE#h zI`m{SlP%;|fO2J|rV5o%uJ!bhkt#je4)FY_aJ(*#N?4aAR%KLmWy$4OO>WLm%Q;j* zt2S0x54f|e$LD(UXk`>vbM@C~a`?jNT#A;J3OQdKoyUoR;;#`MH#b`17gFsAlw~q9--LxXY!jRoC!SB_T~l_f>q z`LDe5-uGdwC}(eA_kQn#DfQm9-iLr(>s>#}4|S~8ddp}7Ez%Y{lJBEMSWv<~9^D|D z42{Tei$jCoOf}d5w^HLMhueUnMN-+XQ)9GF4U1bu!%}+nCz&%(x{@>hVNgMxlt1|D z_tc^+m7Xnr|IbSCYQ6>6mi2d1f^OeOMxGrco-M_*Wkd>tBH+~(LM9!ux{3Hl(nUK& z`4u%jnI?3D&bQZYF%u3dmi@@*wu>~&{JjD9+>A0+2rWWVLF#qmP#B5lxuUqc{lIQE z8XN2~-K~U9G;ji2o)swl(6yPwR>!=2-yPOA+Jzr~kO=2D837OOfXx-!yPO`Sv1LvlnaTvZqK zygCQCh7>p_6JZ{!vLUfFL&upa>8+{TYwwGKZM$*XwnYwx>%U|VK%ZTwVVwr)JLmnF z%H2A8-w@q##}7Q)kF)BwJZ+)Y(VYg#`pHX+oP>z-35e~2rdWCF$|;8yl~bF{>6hNZ zp8Dz*kvjL*A^9SEj1%kH@yUUo!14W8GP2hB$&0{E5;1e;*qVLooH;~n{+QUr&63QG zXOH31^%EH}*7;*-W)fMb6X~a!ua&zV`MvA_inbl0bGL07ccKu*c9%J>^b`3^r(EF< pWMSkFqWBpY4g&9I6r#yh492t$s#R?PH8lIHqL$8SWw=B&`5zLUfQ|qF diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/train_net.cpython-37.pyc" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/__pycache__/train_net.cpython-37.pyc" deleted file mode 100644 index 33ca069eb951b201753c8af629bfaa6ddeed8081..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11571 zcmd^FOKe=%dA_fCKS_@Gk}O}#w##nUIog=0u%*c`Bq# zhm@IeA*7rMDYNC-kTMri=F9UTWg(<2mKQ_H+2%RH`Kj`GK`|OnH_fsso+FKCnit9! z#B;RqZ1cJDbK*JHc)oeDe6jgL`Gw|Ed8t_}7X>}u_(Jo=@{6kC&JHfQFP*66W#^2O zaFQRYrzV~d2 z*|nBwoA+yu+cql=+w*D%wTkW6+AXu$cHGADC$tIMMU7{xu5azPEReem*WI83w^1~BeD(I~4cB)WPi}PDmBUi4NzY;& zk{-WpH^dVfPu=J?{92XOoZZTyOWjstG#q@+;g`Zs$ItsWq=7n6PO!~^K2Yi3*8=5O z?Z<*hU9TH#%a4ess7NI>E}j!2O(IRz(~kNm+K)fhj~@CNYQL^L*5B>j)%uCMMnBn4 z_0#gc^7GnhG1(DoL3%k}f%o|oLwSMunx;Gdu1pFhPPIq?72 zJIeb%3-Uw$K_tTLwrkrS;y1^Qf|1yd)QSh;zzf?~zc zYb%HcNyk`Fg4D5wQBC(#!8B2>pmlwrZlKP?faaEQA`8l~_Oa56Q~p@}Sn(GJXT_SH zJ5ime6MLlgXM&lBf>*!bKQ%ZX6i!t1stukNy%q+h_}2!{OpHQf2mXcnvq8#<;_cUK+v*4p=%2P?F~PX0vipZ!QVP{qZM2WQuDmPZ`rxOMinNqE9jUV1eckEK;f~8fGKPFe9o@BbR4m@Uml^sQxQ*e8*Zr`RsX8D`)2W z#*=f!8oes^TOWKeFoHyQ62%>U|D8K@pI-~m1J+I-yiPTp6ogm7wlzgbn)uvw*+H8% zP5;mh1({D`t=6iVpquTMYq}(BmPPGVR`o86Cu%EdT2gG8OQHC>{3IJmm+h&QkY?tR z6l#|zI+g`~Tk;(@Xm|Wtv(|H&xg;KoveB4~CY^8^AyLX;h+&$R;QN9%8a550V>7!+ zqSPDrBc~&%=F(WGQg_28kxfi=*l1S;^HEVa0=rdl$1E#dW{Kr9muf8^oIqf1%a?0v zwhuNcc4wP?2vYE^I9Tz7rD*6_@V-7@!G>`u4kn6=iaX?czI2M4z2 zugF#_j(cE33wX<&2cvo)uo~u#8M-a6&Gy?ibM6SzC6iXK-Lf0#_EV*1d%x~h{1F>k zMXW=}ZK66Yjhsy}T23`|{(657@EO3-WRR=Z{GH`IZNSi@#j*YkN4llNU`yNW{bDQZ2bN>>_f&+Em-30P^6LQYj1?R^`A z^4(5R+lE4C1+@7jf}}#+=|f$Sf}O$7yMUys1j@UMebs?_g}4NsI#JIm^NNFO{$1^T z0~b*bd;j+@i;0A}uYCo58Xk^3kb}1TGFOj4h;%S5@6VK9EUg^2o9@ak_WBz3`lTik zm#z5icEei{=h14pKDPW9$t#VT=U3Xz&YRz%QPB{iO7Ce|N#5%#SV_C#y}3NDTQoj< zC3;!8CZ%@g70YF8BL6z&i*N9VV9n!n#4Nv7;d;Yu0nlu|vv-q6H?~)=Zf!7HHXd2s zx_Nz-r`PsMD79qHxOr>mDo^d+*(|MHw?t-oYxU~JmbH6hXQy=Cy1lx!w*h`T>l<4< zwY_)4TH9LP-QC#b2^Z(YWdKz?PV~2K?(Onq&8xKluy854L2JPy%x+bYsdOEiM=;E$ z$0I^&l+%RK#G+UY#v=z1Tz)y~frxAzaLu;-c7-R_H%c38rOlmff^44WdgTnYq-X1( z+d_@4o!wpQ+TQjWj%3ZN@H_xm<(}2DTcQ?@Y69Y{ZB?1=@c6!6x%UBXA+{Xn3rpB$ zl|yd$?anGGuAXjtN;V>&buo(@5!K$JXXJC5aHZ*7LF zvA$Yb<*6HcTcu4)tSFB`n6X;L;~DD7vnp+fri#xNX?OIdeKg!f%sGoYZHYaVymOS7 zOC<>r=3=3W=XhG4Eh+SQM95yA7|Cj$lI22>^2A6!a=mg;!$T2HF1hEZ3dG;FrKp*bo;cZ3s=;5RbK^p0p!i2Wam$fS{tV z17iJnKLPA+)OGd`J`jx`5mrD0RzTE`1+fuIPXXacK<_Q^%zi`F0s4Bb7Z*?n(yameyFt!!Ji?V9AxPqu*&ZH z*?P{;52hXhi90bT{s^|fkQQWzw0?F(%Z+F`j0-jp@SQdw`vCk;bYS3rrqPilfpHY- zXVA+}X+)w&jn&cHENaKA8jve`>&II^_U8sb^#Lpe;NQH!zey+cVLZr(_;;#5J;6f* zj|Je^Y00BcrBj1Npx$Xpfq98CF*>Rn#GKfIGC1qRfO^kKZUXgU%t6FS1KneEK*LW_ zZ9@0hGobH42f~jI5k9z{r;(nJ-31FM(1U3BERpXyv0r8YTt4Efvsv^qcwXS{G}QeWdk!jCK&TB!oi3Es7@-1dOE7FVOvq9Rp-uyCwD~mLCRqv0MZN<9 zWI$BpkT6lVh9dGsP>nIZlrVdQzelLsogC3;qFL%*2yS!f_R!C8*_4!_nx!E?jkK9# z+Q6q!Kb3A{G*78<#$y@D{~F z&%HFx&q`?sBZoN&B2_jIX1y3yX|CQpH81jI?owA&Md0gvUzN@)QaTb5<;DMFC#(&|!%kYC)5)cACq-R?p5Labv_r*<+@Qn2$Q)uXQL;?Q z3MDU7@(Lx-Qt~P#U!>#>O1?zN^OVpa*q4!D&bH6i>8U{2w`xO1FsloLwN`CmU$?2| zTa;{40!O$aQL?PbzDm?BN;W9DMhT%^c7u}ZNceOJq3N8|gbC@f9V)#^rSXF<`fb`B zo}d8;gG}}iBGE*r(S{8{Q~oV9D4yX-Jclxy(aGfrA_$quZWH4>l)OX9*C;7dA{Ov2 zJ-tiGdz4s|e4UayCHE*fq~w4SvK-lcN~~J!{NTYoN@yCQ;-FK@t+x(v>BA3?LH!zA%eukt}p{q?FG7jk(h*I%PBp)dssRtTizz^en!-p{)7{}17$Iz~h z4y|g~Z{rLKt-2??>Oyx2y9E9bbH8Oa><94lnEMZ&&_~$7O3iakQTMYHZm3qrzOBzu zw!)|(pV|LR)yhLAhKbQl^iEC{GAPK*@jk5+i7ZsQM{1Vr`YsRz>Dm95W7l)7YNu;1 z?ZZU?-_8ezF5MjBfg%%@vI$pTs~uvF3Km^_XbWgN=vI0TlB4$?l9T=Rdum_ls}EIn z8{47yssQkB1Hfy(PH-)B<0FU=z@1#D$Pxi~?H$_QByc0POqGJBUt^Ph@+j z35$`>l@t&);--j8LiYn=icSrWg_9l2cb>AUE*aj0klrL|l|@BG9>MV7_g)(T6${I0 zjimsc4@7Ckifn?}HgqAa5ynYc$-=Rd@M>#@CCB?&^{w1BhMuR2JN-~>b zzlZ;kZi*ul!*2~g?;#S}2w+y#R|ndO!Y&31?5m5uQIFuhh)^D(Q9n9}Q6A9?&5sWf zDADn(htCF{4L@ENbZlDWk=~E^Nk0|nz~G2c^rL$Z#V3%6zA{T))d&6YlR2NVi(cH$+qqH)|vZMA;bUjwWO-?F_xys6HZ(YfidP z%%yeAd`pG}CzwwNp>e~po_V4f3dj&>Llk~y!%zrPC~_$7erdNNHX!8>_DgIMi7xsK zW$M-K@z^GOK2LhqvGsx(T(@4A(dLiV@Cms%(wnEO4>~(fFbF41md^wWavkSLILN^W z7ZQ60H%BN{6lQ83Y4q^I7c-1J9TIM2Q6#;r@sRfADP&9 ztJXs7Rr zVvh=Yl#r~MkbW5%Ll(1k*%pZuLSy89%Bp!i4Mz~Ug3>yavOaVJ(LehbwMGJG8W}Q7 zguwZGNIhQ}kRVYhSX|c!1_Vw-XW` znt;j?I#ug=h$agDQs`8Hu$mf7Q>YBR;mMyF6sU#BpAmvj$9QHT_3e zNpT`hR0uu_s3I=}+_}M1&_xhiaCAic^Kc+2a7d7Flh1H0z^$az&H6L!Km7A`Q4UUt zM|ulDq#s8a!m4npT=JjupAVuW4&k^-LR=;UB_=3wK}ojG`4@+^AVL%V3&@EY$R)vR zN$_Gc1LTVMeF1?$75th1;^2~4n^X&0w4V~|kQ%&1&XT#|3M^w4;np}>@>d{;mHNx| zSB^E|bSog>F4h!Ik1JSbtZI-sk!zX_vPAV?rS-cT>+5}^A04{q{BsB>DztN`lSS?| znd|4A40_KXWSqMfeXM!2{_EdV5x7?BZ`8lk&x!G6Q3Eb2u{-AmzZT?ZJfGkE@T_&oaqj8&c*XHL|dmmK7doRypwCOAVKs2J=*fTHUnAvu41@${ zAW_!w;(+dNh&tCWia1Jvq_BRdaZIZ>KX{8=TYZgqvL9l1uA@9DRu5|@Lda7%JwNg{ z;iA;(6$7!NQB`@YzW*aQ$5I$S1=4?xS18^V?kN?j`)u#0LX?+Iz1Skv90h}$cI$zR zTzbM;C91(yC6qE57{Z}d^QjQI&+}saDwXv1&^pCV`E*~-zgk`F05Bk7DEWAO3gAj0f^8Hv&o&k(#^e+ zwZ3_aCq%sIty`PxJh`<4UTa@l+uA^U@A}TIlFTrA52>6LqiPOcSSnA202~gSs#SSV zX(04P9U%B)kaFeGP6uGY=dlhBW)p4!p{$Q-V?U&XG&1`ZC8UPgA0XjcrOVHdt_>l8 z2+DPUi+l;+KN6L$Zu6E%Y_1YMv@>Z13*ey1KJ^ zYh8Fac#=YpyVsH5<(j|GrzQgAGTP3g;m8Tf3qJ^t?(S^e-k@b$eaG6kxwCd%=wO~b zv=N@85TY0@pAq3kM13cT*;Rxh8TorR7;7VbS-Q2lxy=h}*EiO_dUI!UyJTJ4+`_z8 zOV<%(r)8L6nez_22L}z;B8Z3$ML3lXHHC0R*k_an`ay&qT@TBfF3;NchtWX1sp0Y| zd4?ye%m6`+yOSk6?tw%vrU~pd_E0bw5Q?yO5QnU__zW%P)m4OqH+F~n!e{MjmAUZj zxR%%5AH7J&a~>|$L|?+kAbf$ITbve{s+<^|G$stl5ys*vAVwBWZVb2>Sy*fgvB+Ih zPK{T930Dq}RSqdw>x3&bo^Lrfn!cPm`%7YY8mS41)FXWM$0$_di}##@##bsm*j_G?sW z;mJy4FfE`lU2;CI)b;VA%<_2sc|=fna!g|1qb7?ZzOu=LT;!>VjFg?@p~~8!$Yb9n zR!$j$Qp^i^8AxkhjLs9cb=4Q zM+oU1x9}E3)3q%2C+I|qmRL)^2_bH(xb{NqNew-|BjQQ;P2yKDP>78;$v5JW7%UcJ4n?Bt=owkEtKac3b`#%VSF(&+hy<#vrssdxXf6SCXxn&U!g5R*NFl zY*N)tTh=x=i_z?50dJf|kX#l4dCWCH_7p5~-UN#vxfM329D)II%5g5q_o`cz_9i2bK&(Erg!W5>uilxe@ zW@+-NTe^H2mVu|{CK^dAsYbX1jg+RSN+WG$V#r`4Eg@NJD2C)>$gq`|m!+h%yBUsc*t#bzxW8<(M{g;s*kzDYL%eY z^q$uIIZYHEI>Dw>tL_9vj}<+L*PgRv56{=zKQ0t^Zoh5hEycOe9FV)TTlukO->w{sM4X_kTztTaI z1m1UK{)4!DM$bRC?;7^4fT2eBbEgI>ViOX9SeWYnA_V zFcs#4>238;!^qnIAUGS$gtN4AO$Bo1j^LeK zINUu4Z4B>kg(KbbJq6THq0O<*I5>>=IsD!JUCCkm1c&ic959j`egO{oaQuivI1-LY z`2Hqo>_UAE^8P2zz=TXAY-CF5(dnJ(6uevn?WxhYucNWo8o$#YUBqZU#2&(7HY!g( z2ihz_4v7*YsUwa5Hkj>R3UfWRlaswO=$(v#*G>~KC@{Z)>-N>;sa#WC`f>#K0RW&?= zU!kh%il#tnXn-ZFYH)>a*`bD8KN;&w=eOjJ@RZU=T2nidtRPpNl&elgHyzqdv9Zq7 zYjywEkUn0GZpCZ$*)NaoMQ1uZL#Nj{8&1>i>Q|}l%!CCAJr_>O=XtF1qu^{f6Vtak z7s4T?zOQtK!YTQjiqGY(*Si1raeowNZ1CTLb~3mYPRFzvoQ>JJ2%Ne4bvCi90ms1T z?fMPbyBJ<3>yT#FOND?^@8rtPaB6PS|>viFS6Y%t$l2nnbEe)V)v z=Ox~l#8U|`JP%ut0=Lgt(i^1_Oa4T|N@%w=@VSIF=!P2n^YEOP49`jKm%~i`7S_KU zUSeloVddbR?(N{+?svl3p4w9(%NQhqJUyfPT}Y&YlU{!>oMkiJ_rsykWae|+8&FsYhJ4z_*e;$BT7fE80|(NL zA~>dZj)^MURrA{Y?dl5r(%c=h()6}#Re#=GbOO8P`ew!U%q_>n|K#ph&>h z%dawb-I^a%nvK>+MPf~IB-Mi}yzP}wBj0tK72EYcnm?wPGpr%(zUl-~@Zei%3LMG} z$M&q0?D_!{sY=sra_shC;z7-;`8y`#_KTWVHRG#4|K)$-vjGKXy{1>*0;gR}i^NuQ z-|+!Xy7rdi`kb`&yHysdU8NVROzH$H=#^G`PUCqjAauM^#|q=4fblUjZTuny{C-UM z6c#vQV&HzTZ{qU&f0K(24Sf@n&M_0A+t2MU>0IZ(q;Q+mxg`~QFt$Xt#r$Ds;V}Zt z7xqhZ=Ttrftb;;-?l@NT@0fGCoEGUVJE-iGYiwT($&eA}Ri=1z?n9AUdGe_I@bQy% zk)huD$Hle9b)LeiVsN9hvc9@@ZxtYsezdw+S}s3b0^)M<)6!abd1-xvLfmTA;likS z+f6<|WJ3|=W@+i+#~bTnkZxRjcf?G~ZT3fHSjqAUJId2Uo{;2Xkfgc4TwGr-t&6i+8T>e{2?W088eR$MG2uolVEO7Y%ui6h!o zgt1uMC<^0IskkEa^^HYg1beI@GBv;E`GM_K9AVI#B@*2Bs*ccI#}h`S&1{i?0c-do zy;#~P-QQSRT@k~ypA%Ax+>`ZEd1LKKsk~ggS6UVtYm0H)b<0(p6N2g4JdW7v?|h$*KRo64j|1&3A!C1vDZr*LN7io3jN;0H8Id~ zc%u!BjC^6+0j#u`;LdZ02jxJzma<2$FJ%BCO|Ndb;?`Os5j1&aM`X5nvrz`0)=Qyz z0Nb3~FGafT`Fm~0AsU-87!x=gk;I8M>_7|_vGb=z+Wq?66_GjiQ6eF|3trgmsz^rm zS){!(bDr0brif%@cSR~P#V`u9P=S?}RAjQmP&9LNKhlyj%~&H<` z0$&)!N}F;jyk^!gbx)94IRZrQkq_NQxN@neD_>G;=okxco0XzmkfgNm-eFmp8{*X! z3As;hk_2*PS}95Z->?v3-IsG3p@HLGiC65o`X)lyng z9o4g1R@LN};CU?rtQj?><0#!~csFUD0u7eA(4RzeeYqJKhG?Fl{X4NEm8fFT57Oe(KP33j$sX}&- zK4jU*4#~9j7w{&55^79+rFL`#z;2*+RV1rOA7m=0KKle=Fr{+|N?1Ds6xi|~2S%6( z5|qqwcmKU084QGpIN_K?3Y!R12+>mvsV`DFjP5FXD>9Kw96>vYFxgEZkxMdzC6Js( zNog91Csso03O)#a4K9LoHxp*UWL1-iR;H5#_jJtt*K#(E5H{`Qqka{VvjGIhX<7s6 z#cgmJBu-&QJ~PMkn7zJ#OUj>)C-athIB~R2Wj^#84j}K zgmS1M8NJ-eB0wGjCGpGklhE&3*d2lbZ`q^gKT+OYM+AP*w&DtMTq#dSdmi^PzG>_ z8V4hrEKgBdSze{ti)aq6-FI!@M{0lt9sgU7_zohKmAL{>650FIJfP+hH6Kv(AvJfX z`2%Y1Qd2}DGRJOII!{ivz#4+oQH_htwn#D<{K^jBC9-SO%u(|MjYuNb3LIt?qMcVr z+7tU1&?Um(xgowuB%e@22V|Z7CQZe4HMw*C8R3%8;WjlDG*&jc>F~HXph!!93^<&% z^oQgK-Z&}*ME}Fl{0hYWJv2%>qh5x|%)-P@lUdetFzy+QvM}>`^dloYs!nPt7}RlT zQZ*Rvf^MLn)A=4&>EAbs(8}v=t(<5kLbQ4rFLoF{^ zALynq2UCU-3{}6M?GB-jsu1AA-8_>nW1sIRAM-6$Uj# z)flE&$Ls`E=|Z)K%s!avPGiRCX&o5+`q{e5e}m@?e&_1vcQs#*Sb}=Nh0j%5Ii5Gi zAvNlR7rV10MMw7xmh9-5RqB`6z%H5HA7F$v(aQ%HSn5#43M9YWVZZkCgWz)aN=%pO zX@5G|F@mf0w@_0;MOaqL;CQH#@I7Ts`Gkz)H8_+c$uFs(Q|Nai?$625?LOz9g7Xk@ z2IPAEZQ0(~O+u#M1ULIM^M@+Sp}wQ}w}C(0T{u10Lyhut)ng~txP>(|K6)CD<$vcI zX7COhLG9;+G_rq->MeZ@jYf@F~7j}cZn$iTJCs(`|>n<)P%T6|6<%kwsl(b^U`}9JllZwk!ft9rqbLtD?3f! z@yz=xE72+XW~1!~R9$lRU|6ZXL&tSYK%p~2}P)S+gMw-*HfPS;GSB zrP3l_rOCmD?LjSNIX6H_Ht58V?*#1@3WU)xv-)I%^2f!cwG)nyDpSI6DDlamni!Ro z;Pjdmky~DUSdP;3rIiP($dzh7pja>ZYYO*pyD78^5?HD)BBS(!a%IPY zoyQf%p4t@@lH~te{3C!Pcm9wNGWUub_dhPLFIgp#EBZciQ{3PZ=S|Kp6M`0OiIEd! zyr?E9@|4am$t*j{@;Mbkgwbpv9Z$Aum0gz-b?_8vSx2Bf^4rAxrbH~R-TzpOtuL)S zTrSBn!DjK(G6AEw;w{3IVZrF5C(9d4r%=S$N;6n$v|Oiw>+Y~fTuMl!>!e5l7kF+! z#s^WPAY%rLIIG){x0mrizq%)jRRSq(9Ibo>R6jtY3{Xy$)icPavO1n>MvokLUc(s9 zd`5?BM}|asnE}U7IU8kh$k$|kh5QNm*(9I^Eq)^J117#X>GHG4wkWfrd@Boj{+Hl8 zr>@N15DA(8%S*_A2B^GoGB6WoYE63Y=22s8G`;!ln(LHnO?eL1#8Jg}o@x)ciT-?4 ze~`sP8jFMk@MSD2ruyg-IDdSNMCQ29e?qv^Z{VJIX~f`(LG)r3-oTDu2TpY~G4{qJ z-Ee+LGgtdFM<)^$R->}X7FFaa9v1!ZQcjEEH|F_Iv6wZ|U+hR3^5w9G`*7)ad`@qp zb-52c%{s1Ulq0oKt;)LcEvTo1b{R{HnQy_3mZKmqD&)QOEqGC2e8): - count1=count - os.rename(Olddir,Olddir[:5]+'_'+str("{:0>6d}".format(count1))+'.jpg')#重命名 - count+=1 \ No newline at end of file diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/test_net.py" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/test_net.py" deleted file mode 100644 index 550b1bf..0000000 --- "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/test_net.py" +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. - -"""Multi-view test a video classification model.""" - -import numpy as np -import os -import pickle -import torch -from iopath.common.file_io import g_pathmgr - -import slowfast.utils.checkpoint as cu -import slowfast.utils.distributed as du -import slowfast.utils.logging as logging -import slowfast.utils.misc as misc -import slowfast.visualization.tensorboard_vis as tb -from slowfast.datasets import loader -from slowfast.models import build_model -from slowfast.utils.meters import AVAMeter, TestMeter - -logger = logging.get_logger(__name__) - - -@torch.no_grad() -def perform_test(test_loader, model, test_meter, cfg, writer=None): - """ - For classification: - Perform mutli-view testing that uniformly samples N clips from a video along - its temporal axis. For each clip, it takes 3 crops to cover the spatial - dimension, followed by averaging the softmax scores across all Nx3 views to - form a video-level prediction. All video predictions are compared to - ground-truth labels and the final testing performance is logged. - For detection: - Perform fully-convolutional testing on the full frames without crop. - Args: - test_loader (loader): video testing loader. - model (model): the pretrained video model to test. - test_meter (TestMeter): testing meters to log and ensemble the testing - results. - cfg (CfgNode): configs. Details can be found in - slowfast/config/defaults.py - writer (TensorboardWriter object, optional): TensorboardWriter object - to writer Tensorboard log. - """ - # Enable eval mode. - model.eval() - test_meter.iter_tic() - - for cur_iter, (inputs, labels, video_idx, meta) in enumerate(test_loader): - if cfg.NUM_GPUS: - # Transfer the data to the current GPU device. - if isinstance(inputs, (list,)): - for i in range(len(inputs)): - inputs[i] = inputs[i].cuda(non_blocking=True) - else: - inputs = inputs.cuda(non_blocking=True) - - # Transfer the data to the current GPU device. - labels = labels.cuda() - video_idx = video_idx.cuda() - for key, val in meta.items(): - if isinstance(val, (list,)): - for i in range(len(val)): - val[i] = val[i].cuda(non_blocking=True) - else: - meta[key] = val.cuda(non_blocking=True) - test_meter.data_toc() - - if cfg.DETECTION.ENABLE: - # Compute the predictions. - preds = model(inputs, meta["boxes"]) - ori_boxes = meta["ori_boxes"] - metadata = meta["metadata"] - - preds = preds.detach().cpu() if cfg.NUM_GPUS else preds.detach() - ori_boxes = ( - ori_boxes.detach().cpu() if cfg.NUM_GPUS else ori_boxes.detach() - ) - metadata = ( - metadata.detach().cpu() if cfg.NUM_GPUS else metadata.detach() - ) - - if cfg.NUM_GPUS > 1: - preds = torch.cat(du.all_gather_unaligned(preds), dim=0) - ori_boxes = torch.cat(du.all_gather_unaligned(ori_boxes), dim=0) - metadata = torch.cat(du.all_gather_unaligned(metadata), dim=0) - - test_meter.iter_toc() - # Update and log stats. - test_meter.update_stats(preds, ori_boxes, metadata) - test_meter.log_iter_stats(None, cur_iter) - else: - # Perform the forward pass. - preds = model(inputs) - - # Gather all the predictions across all the devices to perform ensemble. - if cfg.NUM_GPUS > 1: - preds, labels, video_idx = du.all_gather( - [preds, labels, video_idx] - ) - if cfg.NUM_GPUS: - preds = preds.cpu() - labels = labels.cpu() - video_idx = video_idx.cpu() - - test_meter.iter_toc() - # Update and log stats. - test_meter.update_stats( - preds.detach(), labels.detach(), video_idx.detach() - ) - test_meter.log_iter_stats(cur_iter) - - test_meter.iter_tic() - - # Log epoch stats and print the final testing results. - if not cfg.DETECTION.ENABLE: - all_preds = test_meter.video_preds.clone().detach() - all_labels = test_meter.video_labels - if cfg.NUM_GPUS: - all_preds = all_preds.cpu() - all_labels = all_labels.cpu() - if writer is not None: - writer.plot_eval(preds=all_preds, labels=all_labels) - - if cfg.TEST.SAVE_RESULTS_PATH != "": - save_path = os.path.join(cfg.OUTPUT_DIR, cfg.TEST.SAVE_RESULTS_PATH) - - if du.is_root_proc(): - with g_pathmgr.open(save_path, "wb") as f: - pickle.dump([all_preds, all_labels], f) - - logger.info( - "Successfully saved prediction results to {}".format(save_path) - ) - - test_meter.finalize_metrics() - return test_meter - - -def test(cfg): - """ - Perform multi-view testing on the pretrained video model. - Args: - cfg (CfgNode): configs. Details can be found in - slowfast/config/defaults.py - """ - # Set up environment. - du.init_distributed_training(cfg) - # Set random seed from configs. - np.random.seed(cfg.RNG_SEED) - torch.manual_seed(cfg.RNG_SEED) - - # Setup logging format. - logging.setup_logging(cfg.OUTPUT_DIR) - - # Print config. - logger.info("Test with config:") - logger.info(cfg) - - # Build the video model and print model statistics. - model = build_model(cfg) - if du.is_master_proc() and cfg.LOG_MODEL_INFO: - misc.log_model_info(model, cfg, use_train_input=False) - - cu.load_test_checkpoint(cfg, model) - - # Create video testing loaders. - test_loader = loader.construct_loader(cfg, "test") - logger.info("Testing model for {} iterations".format(len(test_loader))) - - if cfg.DETECTION.ENABLE: - assert cfg.NUM_GPUS == cfg.TEST.BATCH_SIZE or cfg.NUM_GPUS == 0 - test_meter = AVAMeter(len(test_loader), cfg, mode="test") - else: - assert ( - test_loader.dataset.num_videos - % (cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS) - == 0 - ) - # Create meters for multi-view testing. - test_meter = TestMeter( - test_loader.dataset.num_videos - // (cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS), - cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS, - cfg.MODEL.NUM_CLASSES, - len(test_loader), - cfg.DATA.MULTI_LABEL, - cfg.DATA.ENSEMBLE_METHOD, - ) - - # Set up writer for logging to Tensorboard format. - if cfg.TENSORBOARD.ENABLE and du.is_master_proc( - cfg.NUM_GPUS * cfg.NUM_SHARDS - ): - writer = tb.TensorboardWriter(cfg) - else: - writer = None - - # # Perform multi-view test on the entire dataset. - test_meter = perform_test(test_loader, model, test_meter, cfg, writer) - if writer is not None: - writer.close() diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/train_net.py" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/train_net.py" deleted file mode 100644 index 24a861c..0000000 --- "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/train_net.py" +++ /dev/null @@ -1,524 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. - -"""Train a video classification model.""" - -import numpy as np -import pprint -import torch -from fvcore.nn.precise_bn import get_bn_modules, update_bn_stats - -import slowfast.models.losses as losses -import slowfast.models.optimizer as optim -import slowfast.utils.checkpoint as cu -import slowfast.utils.distributed as du -import slowfast.utils.logging as logging -import slowfast.utils.metrics as metrics -import slowfast.utils.misc as misc -import slowfast.visualization.tensorboard_vis as tb -from slowfast.datasets import loader -from slowfast.datasets.mixup import MixUp -from slowfast.models import build_model -from slowfast.utils.meters import AVAMeter, EpochTimer, TrainMeter, ValMeter -from slowfast.utils.multigrid import MultigridSchedule - -logger = logging.get_logger(__name__) - - -def train_epoch( - train_loader, model, optimizer, train_meter, cur_epoch, cfg, writer=None -): - """ - Perform the video training for one epoch. - Args: - train_loader (loader): video training loader. - model (model): the video model to train. - optimizer (optim): the optimizer to perform optimization on the model's - parameters. - train_meter (TrainMeter): training meters to log the training performance. - cur_epoch (int): current epoch of training. - cfg (CfgNode): configs. Details can be found in - slowfast/config/defaults.py - writer (TensorboardWriter, optional): TensorboardWriter object - to writer Tensorboard log. - """ - # Enable train mode. - model.train() - train_meter.iter_tic() - data_size = len(train_loader) - - if cfg.MIXUP.ENABLE: - mixup_fn = MixUp( - mixup_alpha=cfg.MIXUP.ALPHA, - cutmix_alpha=cfg.MIXUP.CUTMIX_ALPHA, - mix_prob=cfg.MIXUP.PROB, - switch_prob=cfg.MIXUP.SWITCH_PROB, - label_smoothing=cfg.MIXUP.LABEL_SMOOTH_VALUE, - num_classes=cfg.MODEL.NUM_CLASSES, - ) - - for cur_iter, (inputs, labels, _, meta) in enumerate(train_loader): - # Transfer the data to the current GPU device. - if cfg.NUM_GPUS: - if isinstance(inputs, (list,)): - for i in range(len(inputs)): - inputs[i] = inputs[i].cuda(non_blocking=True) - else: - inputs = inputs.cuda(non_blocking=True) - labels = labels.cuda() - for key, val in meta.items(): - if isinstance(val, (list,)): - for i in range(len(val)): - val[i] = val[i].cuda(non_blocking=True) - else: - meta[key] = val.cuda(non_blocking=True) - - # Update the learning rate. - lr = optim.get_epoch_lr(cur_epoch + float(cur_iter) / data_size, cfg) - optim.set_lr(optimizer, lr) - - train_meter.data_toc() - if cfg.MIXUP.ENABLE: - samples, labels = mixup_fn(inputs[0], labels) - inputs[0] = samples - - if cfg.DETECTION.ENABLE: - preds = model(inputs, meta["boxes"]) - else: - preds = model(inputs) - # Explicitly declare reduction to mean. - loss_fun = losses.get_loss_func(cfg.MODEL.LOSS_FUNC)(reduction="mean") - - # Compute the loss. - loss = loss_fun(preds, labels) - - # check Nan Loss. - misc.check_nan_losses(loss) - - # Perform the backward pass. - optimizer.zero_grad() - loss.backward() - # Update the parameters. - optimizer.step() - - if cfg.MIXUP.ENABLE: - _top_max_k_vals, top_max_k_inds = torch.topk( - labels, 2, dim=1, largest=True, sorted=True - ) - idx_top1 = torch.arange(labels.shape[0]), top_max_k_inds[:, 0] - idx_top2 = torch.arange(labels.shape[0]), top_max_k_inds[:, 1] - preds = preds.detach() - preds[idx_top1] += preds[idx_top2] - preds[idx_top2] = 0.0 - labels = top_max_k_inds[:, 0] - - if cfg.DETECTION.ENABLE: - if cfg.NUM_GPUS > 1: - loss = du.all_reduce([loss])[0] - loss = loss.item() - - # Update and log stats. - train_meter.update_stats(None, None, None, loss, lr) - # write to tensorboard format if available. - if writer is not None: - writer.add_scalars( - {"Train/loss": loss, "Train/lr": lr}, - global_step=data_size * cur_epoch + cur_iter, - ) - - else: - top1_err, top5_err = None, None - if cfg.DATA.MULTI_LABEL: - # Gather all the predictions across all the devices. - if cfg.NUM_GPUS > 1: - [loss] = du.all_reduce([loss]) - loss = loss.item() - else: - # Compute the errors. - num_topks_correct = metrics.topks_correct(preds, labels, (1, 5)) - top1_err, top5_err = [ - (1.0 - x / preds.size(0)) * 100.0 for x in num_topks_correct - ] - # Gather all the predictions across all the devices. - if cfg.NUM_GPUS > 1: - loss, top1_err, top5_err = du.all_reduce( - [loss, top1_err, top5_err] - ) - - # Copy the stats from GPU to CPU (sync point). - loss, top1_err, top5_err = ( - loss.item(), - top1_err.item(), - top5_err.item(), - ) - - # Update and log stats. - train_meter.update_stats( - top1_err, - top5_err, - loss, - lr, - inputs[0].size(0) - * max( - cfg.NUM_GPUS, 1 - ), # If running on CPU (cfg.NUM_GPUS == 1), use 1 to represent 1 CPU. - ) - # write to tensorboard format if available. - if writer is not None: - writer.add_scalars( - { - "Train/loss": loss, - "Train/lr": lr, - "Train/Top1_err": top1_err, - "Train/Top5_err": top5_err, - }, - global_step=data_size * cur_epoch + cur_iter, - ) - - train_meter.iter_toc() # measure allreduce for this meter - train_meter.log_iter_stats(cur_epoch, cur_iter) - train_meter.iter_tic() - - # Log epoch stats. - train_meter.log_epoch_stats(cur_epoch) - train_meter.reset() - - -@torch.no_grad() -def eval_epoch(val_loader, model, val_meter, cur_epoch, cfg, writer=None): - """ - Evaluate the model on the val set. - Args: - val_loader (loader): data loader to provide validation data. - model (model): model to evaluate the performance. - val_meter (ValMeter): meter instance to record and calculate the metrics. - cur_epoch (int): number of the current epoch of training. - cfg (CfgNode): configs. Details can be found in - slowfast/config/defaults.py - writer (TensorboardWriter, optional): TensorboardWriter object - to writer Tensorboard log. - """ - - # Evaluation mode enabled. The running stats would not be updated. - model.eval() - val_meter.iter_tic() - - for cur_iter, (inputs, labels, _, meta) in enumerate(val_loader): - if cfg.NUM_GPUS: - # Transferthe data to the current GPU device. - if isinstance(inputs, (list,)): - for i in range(len(inputs)): - inputs[i] = inputs[i].cuda(non_blocking=True) - else: - inputs = inputs.cuda(non_blocking=True) - labels = labels.cuda() - for key, val in meta.items(): - if isinstance(val, (list,)): - for i in range(len(val)): - val[i] = val[i].cuda(non_blocking=True) - else: - meta[key] = val.cuda(non_blocking=True) - val_meter.data_toc() - - if cfg.DETECTION.ENABLE: - # Compute the predictions. - preds = model(inputs, meta["boxes"]) - ori_boxes = meta["ori_boxes"] - metadata = meta["metadata"] - - if cfg.NUM_GPUS: - preds = preds.cpu() - ori_boxes = ori_boxes.cpu() - metadata = metadata.cpu() - - if cfg.NUM_GPUS > 1: - preds = torch.cat(du.all_gather_unaligned(preds), dim=0) - ori_boxes = torch.cat(du.all_gather_unaligned(ori_boxes), dim=0) - metadata = torch.cat(du.all_gather_unaligned(metadata), dim=0) - - val_meter.iter_toc() - # Update and log stats. - val_meter.update_stats(preds, ori_boxes, metadata) - - else: - preds = model(inputs) - - if cfg.DATA.MULTI_LABEL: - if cfg.NUM_GPUS > 1: - preds, labels = du.all_gather([preds, labels]) - else: - # Compute the errors. - num_topks_correct = metrics.topks_correct(preds, labels, (1, 5)) - - # Combine the errors across the GPUs. - top1_err, top5_err = [ - (1.0 - x / preds.size(0)) * 100.0 for x in num_topks_correct - ] - if cfg.NUM_GPUS > 1: - top1_err, top5_err = du.all_reduce([top1_err, top5_err]) - - # Copy the errors from GPU to CPU (sync point). - top1_err, top5_err = top1_err.item(), top5_err.item() - - val_meter.iter_toc() - # Update and log stats. - val_meter.update_stats( - top1_err, - top5_err, - inputs[0].size(0) - * max( - cfg.NUM_GPUS, 1 - ), # If running on CPU (cfg.NUM_GPUS == 1), use 1 to represent 1 CPU. - ) - # write to tensorboard format if available. - if writer is not None: - writer.add_scalars( - {"Val/Top1_err": top1_err, "Val/Top5_err": top5_err}, - global_step=len(val_loader) * cur_epoch + cur_iter, - ) - - val_meter.update_predictions(preds, labels) - - val_meter.log_iter_stats(cur_epoch, cur_iter) - val_meter.iter_tic() - - # Log epoch stats. - val_meter.log_epoch_stats(cur_epoch) - # write to tensorboard format if available. - if writer is not None: - if cfg.DETECTION.ENABLE: - writer.add_scalars( - {"Val/mAP": val_meter.full_map}, global_step=cur_epoch - ) - else: - all_preds = [pred.clone().detach() for pred in val_meter.all_preds] - all_labels = [ - label.clone().detach() for label in val_meter.all_labels - ] - if cfg.NUM_GPUS: - all_preds = [pred.cpu() for pred in all_preds] - all_labels = [label.cpu() for label in all_labels] - writer.plot_eval( - preds=all_preds, labels=all_labels, global_step=cur_epoch - ) - - val_meter.reset() - - -def calculate_and_update_precise_bn(loader, model, num_iters=200, use_gpu=True): - """ - Update the stats in bn layers by calculate the precise stats. - Args: - loader (loader): data loader to provide training data. - model (model): model to update the bn stats. - num_iters (int): number of iterations to compute and update the bn stats. - use_gpu (bool): whether to use GPU or not. - """ - - def _gen_loader(): - for inputs, *_ in loader: - if use_gpu: - if isinstance(inputs, (list,)): - for i in range(len(inputs)): - inputs[i] = inputs[i].cuda(non_blocking=True) - else: - inputs = inputs.cuda(non_blocking=True) - yield inputs - - # Update the bn stats. - update_bn_stats(model, _gen_loader(), num_iters) - - -def build_trainer(cfg): - """ - Build training model and its associated tools, including optimizer, - dataloaders and meters. - Args: - cfg (CfgNode): configs. Details can be found in - slowfast/config/defaults.py - Returns: - model (nn.Module): training model. - optimizer (Optimizer): optimizer. - train_loader (DataLoader): training data loader. - val_loader (DataLoader): validatoin data loader. - precise_bn_loader (DataLoader): training data loader for computing - precise BN. - train_meter (TrainMeter): tool for measuring training stats. - val_meter (ValMeter): tool for measuring validation stats. - """ - # Build the video model and print model statistics. - model = build_model(cfg) - if du.is_master_proc() and cfg.LOG_MODEL_INFO: - misc.log_model_info(model, cfg, use_train_input=True) - - # Construct the optimizer. - optimizer = optim.construct_optimizer(model, cfg) - - # Create the video train and val loaders. - train_loader = loader.construct_loader(cfg, "train") - val_loader = loader.construct_loader(cfg, "val") - precise_bn_loader = loader.construct_loader( - cfg, "train", is_precise_bn=True - ) - # Create meters. - train_meter = TrainMeter(len(train_loader), cfg) - val_meter = ValMeter(len(val_loader), cfg) - - return ( - model, - optimizer, - train_loader, - val_loader, - precise_bn_loader, - train_meter, - val_meter, - ) - - -def train(cfg): - """ - Train a video model for many epochs on train set and evaluate it on val set. - Args: - cfg (CfgNode): configs. Details can be found in - slowfast/config/defaults.py - """ - # Set up environment. - du.init_distributed_training(cfg) - # Set random seed from configs. - np.random.seed(cfg.RNG_SEED) - torch.manual_seed(cfg.RNG_SEED) - - # Setup logging format. - logging.setup_logging(cfg.OUTPUT_DIR) - - # Init multigrid. - multigrid = None - if cfg.MULTIGRID.LONG_CYCLE or cfg.MULTIGRID.SHORT_CYCLE: - multigrid = MultigridSchedule() - cfg = multigrid.init_multigrid(cfg) - if cfg.MULTIGRID.LONG_CYCLE: - cfg, _ = multigrid.update_long_cycle(cfg, cur_epoch=0) - # Print config. - logger.info("Train with config:") - logger.info(pprint.pformat(cfg)) - - # Build the video model and print model statistics. - model = build_model(cfg) - if du.is_master_proc() and cfg.LOG_MODEL_INFO: - misc.log_model_info(model, cfg, use_train_input=True) - - # Construct the optimizer. - optimizer = optim.construct_optimizer(model, cfg) - - # Load a checkpoint to resume training if applicable. - start_epoch = cu.load_train_checkpoint(cfg, model, optimizer) - - # Create the video train and val loaders. - train_loader = loader.construct_loader(cfg, "train") - val_loader = loader.construct_loader(cfg, "val") - precise_bn_loader = ( - loader.construct_loader(cfg, "train", is_precise_bn=True) - if cfg.BN.USE_PRECISE_STATS - else None - ) - - # Create meters. - if cfg.DETECTION.ENABLE: - train_meter = AVAMeter(len(train_loader), cfg, mode="train") - val_meter = AVAMeter(len(val_loader), cfg, mode="val") - else: - train_meter = TrainMeter(len(train_loader), cfg) - val_meter = ValMeter(len(val_loader), cfg) - - # set up writer for logging to Tensorboard format. - if cfg.TENSORBOARD.ENABLE and du.is_master_proc( - cfg.NUM_GPUS * cfg.NUM_SHARDS - ): - writer = tb.TensorboardWriter(cfg) - else: - writer = None - - # Perform the training loop. - logger.info("Start epoch: {}".format(start_epoch + 1)) - - epoch_timer = EpochTimer() - - for cur_epoch in range(start_epoch, cfg.SOLVER.MAX_EPOCH): - if cfg.MULTIGRID.LONG_CYCLE: - cfg, changed = multigrid.update_long_cycle(cfg, cur_epoch) - if changed: - ( - model, - optimizer, - train_loader, - val_loader, - precise_bn_loader, - train_meter, - val_meter, - ) = build_trainer(cfg) - - # Load checkpoint. - if cu.has_checkpoint(cfg.OUTPUT_DIR): - last_checkpoint = cu.get_last_checkpoint(cfg.OUTPUT_DIR) - assert "{:05d}.pyth".format(cur_epoch) in last_checkpoint - else: - last_checkpoint = cfg.TRAIN.CHECKPOINT_FILE_PATH - logger.info("Load from {}".format(last_checkpoint)) - cu.load_checkpoint( - last_checkpoint, model, cfg.NUM_GPUS > 1, optimizer - ) - - # Shuffle the dataset. - loader.shuffle_dataset(train_loader, cur_epoch) - - # Train for one epoch. - epoch_timer.epoch_tic() - train_epoch( - train_loader, model, optimizer, train_meter, cur_epoch, cfg, writer - ) - epoch_timer.epoch_toc() - logger.info( - f"Epoch {cur_epoch} takes {epoch_timer.last_epoch_time():.2f}s. Epochs " - f"from {start_epoch} to {cur_epoch} take " - f"{epoch_timer.avg_epoch_time():.2f}s in average and " - f"{epoch_timer.median_epoch_time():.2f}s in median." - ) - logger.info( - f"For epoch {cur_epoch}, each iteraction takes " - f"{epoch_timer.last_epoch_time()/len(train_loader):.2f}s in average. " - f"From epoch {start_epoch} to {cur_epoch}, each iteraction takes " - f"{epoch_timer.avg_epoch_time()/len(train_loader):.2f}s in average." - ) - - is_checkp_epoch = cu.is_checkpoint_epoch( - cfg, - cur_epoch, - None if multigrid is None else multigrid.schedule, - ) - is_eval_epoch = misc.is_eval_epoch( - cfg, cur_epoch, None if multigrid is None else multigrid.schedule - ) - - # Compute precise BN stats. - if ( - (is_checkp_epoch or is_eval_epoch) - and cfg.BN.USE_PRECISE_STATS - and len(get_bn_modules(model)) > 0 - ): - calculate_and_update_precise_bn( - precise_bn_loader, - model, - min(cfg.BN.NUM_BATCHES_PRECISE, len(precise_bn_loader)), - cfg.NUM_GPUS > 0, - ) - _ = misc.aggregate_sub_bn_stats(model) - - # Save a checkpoint. - if is_checkp_epoch: - cu.save_checkpoint(cfg.OUTPUT_DIR, model, optimizer, cur_epoch, cfg) - # Evaluate the model on validation set. - if is_eval_epoch: - eval_epoch(val_loader, model, val_meter, cur_epoch, cfg, writer) - - if writer is not None: - writer.close() diff --git "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/visualization.py" "b/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/visualization.py" deleted file mode 100644 index 905863b..0000000 --- "a/code/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/visualization.py" +++ /dev/null @@ -1,347 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. - -import numpy as np -import pickle -import torch -import tqdm -from iopath.common.file_io import g_pathmgr - -import slowfast.datasets.utils as data_utils -import slowfast.utils.checkpoint as cu -import slowfast.utils.distributed as du -import slowfast.utils.logging as logging -import slowfast.utils.misc as misc -import slowfast.visualization.tensorboard_vis as tb -from slowfast.datasets import loader -from slowfast.models import build_model -from slowfast.visualization.gradcam_utils import GradCAM -from slowfast.visualization.prediction_vis import WrongPredictionVis -from slowfast.visualization.utils import ( - GetWeightAndActivation, - process_layer_index_data, -) -from slowfast.visualization.video_visualizer import VideoVisualizer - -logger = logging.get_logger(__name__) - - -def run_visualization(vis_loader, model, cfg, writer=None): - """ - Run model visualization (weights, activations and model inputs) and visualize - them on Tensorboard. - Args: - vis_loader (loader): video visualization loader. - model (model): the video model to visualize. - cfg (CfgNode): configs. Details can be found in - slowfast/config/defaults.py - writer (TensorboardWriter, optional): TensorboardWriter object - to writer Tensorboard log. - """ - n_devices = cfg.NUM_GPUS * cfg.NUM_SHARDS - prefix = "module/" if n_devices > 1 else "" - # Get a list of selected layer names and indexing. - layer_ls, indexing_dict = process_layer_index_data( - cfg.TENSORBOARD.MODEL_VIS.LAYER_LIST, layer_name_prefix=prefix - ) - logger.info("Start Model Visualization.") - # Register hooks for activations. - model_vis = GetWeightAndActivation(model, layer_ls) - - if writer is not None and cfg.TENSORBOARD.MODEL_VIS.MODEL_WEIGHTS: - layer_weights = model_vis.get_weights() - writer.plot_weights_and_activations( - layer_weights, tag="Layer Weights/", heat_map=False - ) - - video_vis = VideoVisualizer( - cfg.MODEL.NUM_CLASSES, - cfg.TENSORBOARD.CLASS_NAMES_PATH, - cfg.TENSORBOARD.MODEL_VIS.TOPK_PREDS, - cfg.TENSORBOARD.MODEL_VIS.COLORMAP, - ) - if n_devices > 1: - grad_cam_layer_ls = [ - "module/" + layer - for layer in cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST - ] - else: - grad_cam_layer_ls = cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST - - if cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE: - gradcam = GradCAM( - model, - target_layers=grad_cam_layer_ls, - data_mean=cfg.DATA.MEAN, - data_std=cfg.DATA.STD, - colormap=cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.COLORMAP, - ) - logger.info("Finish drawing weights.") - global_idx = -1 - for inputs, labels, _, meta in tqdm.tqdm(vis_loader): - if cfg.NUM_GPUS: - # Transfer the data to the current GPU device. - if isinstance(inputs, (list,)): - for i in range(len(inputs)): - inputs[i] = inputs[i].cuda(non_blocking=True) - else: - inputs = inputs.cuda(non_blocking=True) - labels = labels.cuda() - for key, val in meta.items(): - if isinstance(val, (list,)): - for i in range(len(val)): - val[i] = val[i].cuda(non_blocking=True) - else: - meta[key] = val.cuda(non_blocking=True) - - if cfg.DETECTION.ENABLE: - activations, preds = model_vis.get_activations( - inputs, meta["boxes"] - ) - else: - activations, preds = model_vis.get_activations(inputs) - if cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE: - if cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.USE_TRUE_LABEL: - inputs, preds = gradcam(inputs, labels=labels) - else: - inputs, preds = gradcam(inputs) - if cfg.NUM_GPUS: - inputs = du.all_gather_unaligned(inputs) - activations = du.all_gather_unaligned(activations) - preds = du.all_gather_unaligned(preds) - if isinstance(inputs[0], list): - for i in range(len(inputs)): - for j in range(len(inputs[0])): - inputs[i][j] = inputs[i][j].cpu() - else: - inputs = [inp.cpu() for inp in inputs] - preds = [pred.cpu() for pred in preds] - else: - inputs, activations, preds = [inputs], [activations], [preds] - - boxes = [None] * max(n_devices, 1) - if cfg.DETECTION.ENABLE and cfg.NUM_GPUS: - boxes = du.all_gather_unaligned(meta["boxes"]) - boxes = [box.cpu() for box in boxes] - - if writer is not None: - total_vids = 0 - for i in range(max(n_devices, 1)): - cur_input = inputs[i] - cur_activations = activations[i] - cur_batch_size = cur_input[0].shape[0] - cur_preds = preds[i] - cur_boxes = boxes[i] - for cur_batch_idx in range(cur_batch_size): - global_idx += 1 - total_vids += 1 - if ( - cfg.TENSORBOARD.MODEL_VIS.INPUT_VIDEO - or cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE - ): - for path_idx, input_pathway in enumerate(cur_input): - if cfg.TEST.DATASET == "ava" and cfg.AVA.BGR: - video = input_pathway[ - cur_batch_idx, [2, 1, 0], ... - ] - else: - video = input_pathway[cur_batch_idx] - - if not cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE: - # Permute to (T, H, W, C) from (C, T, H, W). - video = video.permute(1, 2, 3, 0) - video = data_utils.revert_tensor_normalize( - video, cfg.DATA.MEAN, cfg.DATA.STD - ) - else: - # Permute from (T, C, H, W) to (T, H, W, C) - video = video.permute(0, 2, 3, 1) - bboxes = ( - None if cur_boxes is None else cur_boxes[:, 1:] - ) - cur_prediction = ( - cur_preds - if cfg.DETECTION.ENABLE - else cur_preds[cur_batch_idx] - ) - video = video_vis.draw_clip( - video, cur_prediction, bboxes=bboxes - ) - video = ( - torch.from_numpy(np.array(video)) - .permute(0, 3, 1, 2) - .unsqueeze(0) - ) - writer.add_video( - video, - tag="Input {}/Pathway {}".format( - global_idx, path_idx + 1 - ), - ) - if cfg.TENSORBOARD.MODEL_VIS.ACTIVATIONS: - writer.plot_weights_and_activations( - cur_activations, - tag="Input {}/Activations: ".format(global_idx), - batch_idx=cur_batch_idx, - indexing_dict=indexing_dict, - ) - - -def perform_wrong_prediction_vis(vis_loader, model, cfg): - """ - Visualize video inputs with wrong predictions on Tensorboard. - Args: - vis_loader (loader): video visualization loader. - model (model): the video model to visualize. - cfg (CfgNode): configs. Details can be found in - slowfast/config/defaults.py - """ - wrong_prediction_visualizer = WrongPredictionVis(cfg=cfg) - for batch_idx, (inputs, labels, _, _) in tqdm.tqdm(enumerate(vis_loader)): - if cfg.NUM_GPUS: - # Transfer the data to the current GPU device. - if isinstance(inputs, (list,)): - for i in range(len(inputs)): - inputs[i] = inputs[i].cuda(non_blocking=True) - else: - inputs = inputs.cuda(non_blocking=True) - labels = labels.cuda() - - # Some model modify the original input. - inputs_clone = [inp.clone() for inp in inputs] - - preds = model(inputs) - - if cfg.NUM_GPUS > 1: - preds, labels = du.all_gather([preds, labels]) - if isinstance(inputs_clone, (list,)): - inputs_clone = du.all_gather(inputs_clone) - else: - inputs_clone = du.all_gather([inputs_clone])[0] - - if cfg.NUM_GPUS: - # Transfer the data to the current CPU device. - labels = labels.cpu() - preds = preds.cpu() - if isinstance(inputs_clone, (list,)): - for i in range(len(inputs_clone)): - inputs_clone[i] = inputs_clone[i].cpu() - else: - inputs_clone = inputs_clone.cpu() - - # If using CPU (NUM_GPUS = 0), 1 represent 1 CPU. - n_devices = max(cfg.NUM_GPUS, 1) - for device_idx in range(1, n_devices + 1): - wrong_prediction_visualizer.visualize_vid( - video_input=inputs_clone, - labels=labels, - preds=preds.detach().clone(), - batch_idx=device_idx * batch_idx, - ) - - logger.info( - "Class indices with wrong predictions: {}".format( - sorted(wrong_prediction_visualizer.wrong_class_prediction) - ) - ) - wrong_prediction_visualizer.clean() - - -def visualize(cfg): - """ - Perform layer weights and activations visualization on the model. - Args: - cfg (CfgNode): configs. Details can be found in - slowfast/config/defaults.py - """ - if cfg.TENSORBOARD.ENABLE and ( - cfg.TENSORBOARD.MODEL_VIS.ENABLE - or cfg.TENSORBOARD.WRONG_PRED_VIS.ENABLE - ): - # Set up environment. - du.init_distributed_training(cfg) - # Set random seed from configs. - np.random.seed(cfg.RNG_SEED) - torch.manual_seed(cfg.RNG_SEED) - - # Setup logging format. - logging.setup_logging(cfg.OUTPUT_DIR) - - # Print config. - logger.info("Model Visualization with config:") - logger.info(cfg) - - # Build the video model and print model statistics. - model = build_model(cfg) - model.eval() - if du.is_master_proc() and cfg.LOG_MODEL_INFO: - misc.log_model_info(model, cfg, use_train_input=False) - - cu.load_test_checkpoint(cfg, model) - - # Create video testing loaders. - vis_loader = loader.construct_loader(cfg, "test") - - if cfg.DETECTION.ENABLE: - assert cfg.NUM_GPUS == cfg.TEST.BATCH_SIZE or cfg.NUM_GPUS == 0 - - # Set up writer for logging to Tensorboard format. - if du.is_master_proc(cfg.NUM_GPUS * cfg.NUM_SHARDS): - writer = tb.TensorboardWriter(cfg) - else: - writer = None - if cfg.TENSORBOARD.PREDICTIONS_PATH != "": - assert not cfg.DETECTION.ENABLE, "Detection is not supported." - logger.info( - "Visualizing class-level performance from saved results..." - ) - if writer is not None: - with g_pathmgr.open( - cfg.TENSORBOARD.PREDICTIONS_PATH, "rb" - ) as f: - preds, labels = pickle.load(f, encoding="latin1") - - writer.plot_eval(preds, labels) - - if cfg.TENSORBOARD.MODEL_VIS.ENABLE: - if cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE: - assert ( - not cfg.DETECTION.ENABLE - ), "Detection task is currently not supported for Grad-CAM visualization." - if cfg.MODEL.ARCH in cfg.MODEL.SINGLE_PATHWAY_ARCH: - assert ( - len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST) == 1 - ), "The number of chosen CNN layers must be equal to the number of pathway(s), given {} layer(s).".format( - len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST) - ) - elif cfg.MODEL.ARCH in cfg.MODEL.MULTI_PATHWAY_ARCH: - assert ( - len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST) == 2 - ), "The number of chosen CNN layers must be equal to the number of pathway(s), given {} layer(s).".format( - len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST) - ) - else: - raise NotImplementedError( - "Model arch {} is not in {}".format( - cfg.MODEL.ARCH, - cfg.MODEL.SINGLE_PATHWAY_ARCH - + cfg.MODEL.MULTI_PATHWAY_ARCH, - ) - ) - logger.info( - "Visualize model analysis for {} iterations".format( - len(vis_loader) - ) - ) - # Run visualization on the model - run_visualization(vis_loader, model, cfg, writer) - if cfg.TENSORBOARD.WRONG_PRED_VIS.ENABLE: - logger.info( - "Visualize Wrong Predictions for {} iterations".format( - len(vis_loader) - ) - ) - perform_wrong_prediction_vis(vis_loader, model, cfg) - - if writer is not None: - writer.close() -- Gitee From ebc87930dac7dc5ca37095ae666a4903756c0b9d Mon Sep 17 00:00:00 2001 From: Aurora_zxy <9166856+zombie233@user.noreply.gitee.com> Date: Sat, 10 Jul 2021 12:32:43 +0000 Subject: [PATCH 06/10] =?UTF-8?q?=E6=96=B0=E5=BB=BA=202019302110407-?= =?UTF-8?q?=E5=BC=A0=E7=BF=94=E5=AE=87-=E5=8A=A8=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.keep" | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/.keep" diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/.keep" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/.keep" new file mode 100644 index 0000000..e69de29 -- Gitee From 4815b27b9b31eb7d760bc16c86518ad351f63020 Mon Sep 17 00:00:00 2001 From: Aurora_zxy <9166856+zombie233@user.noreply.gitee.com> Date: Sat, 10 Jul 2021 13:00:28 +0000 Subject: [PATCH 07/10] =?UTF-8?q?=E6=96=B0=E5=BB=BA=20=E5=BC=A0=E7=BF=94?= =?UTF-8?q?=E5=AE=87-=E5=8A=A8=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.keep" | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 "code/\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/.keep" diff --git "a/code/\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/.keep" "b/code/\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/.keep" new file mode 100644 index 0000000..e69de29 -- Gitee From fe387200ccb64f57fedeacc050131c81cedb9ca5 Mon Sep 17 00:00:00 2001 From: Aurora_zxy <9166856+zombie233@user.noreply.gitee.com> Date: Sat, 10 Jul 2021 13:08:57 +0000 Subject: [PATCH 08/10] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6=20co?= =?UTF-8?q?de/=E5=BC=A0=E7=BF=94=E5=AE=87-=E5=8A=A8=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.keep" | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 "code/\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/.keep" diff --git "a/code/\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/.keep" "b/code/\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/.keep" deleted file mode 100644 index e69de29..0000000 -- Gitee From 81fa6b455be56385d3cb42ce06ecf05798b2973d Mon Sep 17 00:00:00 2001 From: 15822575186 Date: Sat, 10 Jul 2021 22:01:39 +0800 Subject: [PATCH 09/10] code --- Course_MLPR | 1 + 1 file changed, 1 insertion(+) create mode 160000 Course_MLPR diff --git a/Course_MLPR b/Course_MLPR new file mode 160000 index 0000000..fe38720 --- /dev/null +++ b/Course_MLPR @@ -0,0 +1 @@ +Subproject commit fe387200ccb64f57fedeacc050131c81cedb9ca5 -- Gitee From 85eb812222e9633a33ab3237ca13397080c8a3c9 Mon Sep 17 00:00:00 2001 From: 15822575186 Date: Sat, 10 Jul 2021 22:40:21 +0800 Subject: [PATCH 10/10] code --- .../configs/AVA/SLOWFAST_32x2_R50_SHORT.yaml" | 78 ++ .../AVA/SLOWFAST_32x2_R50_SHORT3.yaml" | 91 ++ .../AVA/SLOWFAST_32x2_R50_SHORT5.yaml" | 84 ++ .../configs/AVA/SLOW_8x8_R50_SHORT.yaml" | 72 ++ .../AVA/c2/SLOWFAST_32x2_R101_50_50.yaml" | 68 ++ .../c2/SLOWFAST_32x2_R101_50_50_v2.1.yaml" | 72 ++ .../configs/AVA/c2/SLOWFAST_32x2_R50.yaml" | 68 ++ .../AVA/c2/SLOWFAST_64x2_R101_50_50.yaml" | 68 ++ .../configs/AVA/c2/SLOW_8x8_R50.yaml" | 62 + .../configs/Charades/SLOWFAST_16x8_R50.yaml" | 75 ++ .../SLOWFAST_16x8_R50_multigrid.yaml" | 78 ++ .../pytorchvideo/SLOWFAST_8x8_R50.yaml" | 74 ++ .../Charades/pytorchvideo/SLOW_8x8_R50.yaml" | 66 ++ .../configs/ImageNet/MVIT_B_16_CONV.yaml" | 81 ++ .../configs/ImageNet/RES_R50.yaml" | 60 + .../configs/Kinetics/C2D_8x8_R50.yaml" | 55 + .../configs/Kinetics/C2D_8x8_R50_IN1K.yaml" | 58 + .../configs/Kinetics/C2D_NLN_8x8_R50.yaml" | 55 + .../Kinetics/C2D_NLN_8x8_R50_IN1K.yaml" | 58 + .../configs/Kinetics/I3D_8x8_R101.yaml" | 55 + .../configs/Kinetics/I3D_8x8_R50.yaml" | 55 + .../configs/Kinetics/I3D_8x8_R50_IN1K.yaml" | 58 + .../configs/Kinetics/I3D_NLN_8x8_R101.yaml" | 55 + .../configs/Kinetics/I3D_NLN_8x8_R50.yaml" | 55 + .../Kinetics/I3D_NLN_8x8_R50_IN1K.yaml" | 58 + .../configs/Kinetics/MVIT_B_16x4_CONV.yaml" | 92 ++ .../configs/Kinetics/SLOWFAST_4x16_R50.yaml" | 62 + .../configs/Kinetics/SLOWFAST_8x8_R50.yaml" | 62 + .../Kinetics/SLOWFAST_8x8_R50_stepwise.yaml" | 64 ++ .../SLOWFAST_8x8_R50_stepwise_multigrid.yaml" | 67 ++ .../Kinetics/SLOWFAST_NLN_4x16_R50.yaml" | 62 + .../Kinetics/SLOWFAST_NLN_8x8_R50.yaml" | 62 + .../configs/Kinetics/SLOW_4x16_R50.yaml" | 55 + .../configs/Kinetics/SLOW_8x8_R50.yaml" | 55 + .../configs/Kinetics/SLOW_NLN_4x16_R50.yaml" | 55 + .../configs/Kinetics/SLOW_NLN_8x8_R50.yaml" | 55 + .../configs/Kinetics/X3D_L.yaml" | 58 + .../configs/Kinetics/X3D_M.yaml" | 58 + .../configs/Kinetics/X3D_S.yaml" | 58 + .../configs/Kinetics/X3D_XS.yaml" | 58 + .../Kinetics/c2/C2D_NOPOOL_8x8_R50.yaml" | 57 + .../configs/Kinetics/c2/I3D_8x8_R50.yaml" | 57 + .../configs/Kinetics/c2/I3D_NLN_8x8_R50.yaml" | 57 + .../c2/SLOWFAST_16x8_R101_50_50.yaml" | 64 ++ .../Kinetics/c2/SLOWFAST_4x16_R50.yaml" | 64 ++ .../c2/SLOWFAST_8x8_R101_101_101.yaml" | 64 ++ .../c2/SLOWFAST_8x8_R101_50_101.yaml" | 64 ++ .../Kinetics/c2/SLOWFAST_8x8_R101_50_50.yaml" | 64 ++ .../Kinetics/c2/SLOWFAST_8x8_R50.yaml" | 64 ++ .../c2/SLOWFAST_NLN_16x8_R101_50_50.yaml" | 64 ++ .../configs/Kinetics/c2/SLOW_4x16_R50.yaml" | 57 + .../configs/Kinetics/c2/SLOW_8x8_R50.yaml" | 57 + .../Kinetics/pytorchvideo/C2D_8x8_R50.yaml" | 59 + .../Kinetics/pytorchvideo/CSN_32x2_R101.yaml" | 58 + .../Kinetics/pytorchvideo/I3D_8x8_R50.yaml" | 61 + .../pytorchvideo/R2PLUS1D_16x4_R50.yaml" | 58 + .../SLOWFAST_16x8_R101_50_50.yaml" | 66 ++ .../pytorchvideo/SLOWFAST_4x16_R50.yaml" | 68 ++ .../pytorchvideo/SLOWFAST_8x8_R101.yaml" | 68 ++ .../pytorchvideo/SLOWFAST_8x8_R50.yaml" | 68 ++ .../Kinetics/pytorchvideo/SLOW_4x16_R50.yaml" | 61 + .../Kinetics/pytorchvideo/SLOW_8x8_R50.yaml" | 61 + .../configs/Kinetics/pytorchvideo/X3D_L.yaml" | 55 + .../configs/Kinetics/pytorchvideo/X3D_M.yaml" | 59 + .../configs/Kinetics/pytorchvideo/X3D_S.yaml" | 60 + .../Kinetics/pytorchvideo/X3D_XS.yaml" | 58 + .../configs/SSv2/SLOWFAST_16x8_R50.yaml" | 73 ++ .../SSv2/SLOWFAST_16x8_R50_multigrid.yaml" | 76 ++ .../SSv2/pytorchvideo/SLOWFAST_8x8_R50.yaml" | 71 ++ .../SSv2/pytorchvideo/SLOW_8x8_R50.yaml" | 63 + .../demo/AVA/Kinetics/SLOWFAST_8x8_R50.yaml" | 67 ++ .../demo/AVA/SLOWFAST_32x2_R101_50_50.yaml" | 80 ++ .../demo/AVA/ava.json" | 1 + .../slowfast/__init__.py" | 6 + .../slowfast/config/__init__.py" | 2 + .../slowfast/config/custom_config.py" | 9 + .../slowfast/config/defaults.py" | 960 ++++++++++++++++ .../slowfast/datasets/DATASET.md" | 165 +++ .../slowfast/datasets/__init__.py" | 13 + .../slowfast/datasets/ava_dataset.py" | 428 +++++++ .../slowfast/datasets/ava_helper.py" | 236 ++++ .../slowfast/datasets/build.py" | 31 + .../slowfast/datasets/charades.py" | 258 +++++ .../slowfast/datasets/cv2_transform.py" | 797 +++++++++++++ .../slowfast/datasets/decoder.py" | 390 +++++++ .../slowfast/datasets/imagenet.py" | 203 ++++ .../slowfast/datasets/kinetics.py" | 408 +++++++ .../slowfast/datasets/loader.py" | 196 ++++ .../slowfast/datasets/mixup.py" | 189 +++ .../slowfast/datasets/multigrid_helper.py" | 87 ++ .../slowfast/datasets/ptv_datasets.py" | 594 ++++++++++ .../slowfast/datasets/rand_augment.py" | 533 +++++++++ .../slowfast/datasets/random_erasing.py" | 180 +++ .../slowfast/datasets/ssv2.py" | 283 +++++ .../slowfast/datasets/transform.py" | 894 +++++++++++++++ .../slowfast/datasets/utils.py" | 347 ++++++ .../slowfast/datasets/video_container.py" | 29 + .../slowfast/models/__init__.py" | 17 + .../slowfast/models/attention.py" | 314 +++++ .../slowfast/models/batchnorm_helper.py" | 218 ++++ .../slowfast/models/build.py" | 53 + .../slowfast/models/common.py" | 70 ++ .../models/custom_video_model_builder.py" | 5 + .../slowfast/models/head_helper.py" | 408 +++++++ .../slowfast/models/losses.py" | 51 + .../slowfast/models/nonlocal_helper.py" | 148 +++ .../slowfast/models/operators.py" | 82 ++ .../slowfast/models/optimizer.py" | 120 ++ .../slowfast/models/ptv_model_builder.py" | 701 ++++++++++++ .../slowfast/models/resnet_helper.py" | 725 ++++++++++++ .../slowfast/models/stem_helper.py" | 320 ++++++ .../slowfast/models/utils.py" | 21 + .../slowfast/models/video_model_builder.py" | 1016 +++++++++++++++++ .../slowfast/utils/__init__.py" | 2 + .../slowfast/utils/ava_eval_helper.py" | 304 +++++ .../slowfast/utils/ava_evaluation/README.md" | 1 + .../utils/ava_evaluation/__init__.py" | 0 ..._list_v2.1_for_activitynet_2018.pbtxt.txt" | 240 ++++ .../utils/ava_evaluation/label_map_util.py" | 187 +++ .../slowfast/utils/ava_evaluation/metrics.py" | 154 +++ .../utils/ava_evaluation/np_box_list.py" | 143 +++ .../utils/ava_evaluation/np_box_list_ops.py" | 593 ++++++++++ .../utils/ava_evaluation/np_box_mask_list.py" | 73 ++ .../ava_evaluation/np_box_mask_list_ops.py" | 428 +++++++ .../utils/ava_evaluation/np_box_ops.py" | 108 ++ .../utils/ava_evaluation/np_mask_ops.py" | 130 +++ .../object_detection_evaluation.py" | 832 ++++++++++++++ .../ava_evaluation/per_image_evaluation.py" | 453 ++++++++ .../utils/ava_evaluation/standard_fields.py" | 225 ++++ .../slowfast/utils/benchmark.py" | 103 ++ .../slowfast/utils/bn_helper.py" | 77 ++ .../slowfast/utils/c2_model_loading.py" | 120 ++ .../slowfast/utils/checkpoint.py" | 514 +++++++++ .../slowfast/utils/distributed.py" | 309 +++++ .../slowfast/utils/env.py" | 15 + .../slowfast/utils/logging.py" | 99 ++ .../slowfast/utils/lr_policy.py" | 94 ++ .../slowfast/utils/meters.py" | 823 +++++++++++++ .../slowfast/utils/metrics.py" | 64 ++ .../slowfast/utils/misc.py" | 383 +++++++ .../slowfast/utils/multigrid.py" | 240 ++++ .../slowfast/utils/multiprocessing.py" | 62 + .../slowfast/utils/parser.py" | 94 ++ .../slowfast/utils/weight_init_helper.py" | 44 + .../slowfast/visualization/__init__.py" | 2 + .../visualization/async_predictor.py" | 319 ++++++ .../ava_demo_precomputed_boxes.py" | 467 ++++++++ .../slowfast/visualization/demo_loader.py" | 367 ++++++ .../slowfast/visualization/gradcam_utils.py" | 178 +++ .../slowfast/visualization/prediction_vis.py" | 135 +++ .../slowfast/visualization/predictor.py" | 207 ++++ .../visualization/tensorboard_vis.py" | 429 +++++++ .../slowfast/visualization/utils.py" | 375 ++++++ .../visualization/video_visualizer.py" | 677 +++++++++++ .../tools/benchmark.py" | 25 + .../tools/demo_net.py" | 119 ++ .../tools/run_net.py" | 49 + .../tools/ss.py" | 19 + .../tools/test_net.py" | 202 ++++ .../tools/train_net.py" | 524 +++++++++ .../tools/visualization.py" | 347 ++++++ 161 files changed, 27468 insertions(+) create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOWFAST_32x2_R50_SHORT.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOWFAST_32x2_R50_SHORT3.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOWFAST_32x2_R50_SHORT5.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOW_8x8_R50_SHORT.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_32x2_R101_50_50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_32x2_R101_50_50_v2.1.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_32x2_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_64x2_R101_50_50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOW_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/SLOWFAST_16x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/SLOWFAST_16x8_R50_multigrid.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/pytorchvideo/SLOWFAST_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/pytorchvideo/SLOW_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/ImageNet/MVIT_B_16_CONV.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/ImageNet/RES_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_8x8_R50_IN1K.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_NLN_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_NLN_8x8_R50_IN1K.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_8x8_R101.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_8x8_R50_IN1K.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_NLN_8x8_R101.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_NLN_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_NLN_8x8_R50_IN1K.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/MVIT_B_16x4_CONV.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_4x16_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_8x8_R50_stepwise.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_8x8_R50_stepwise_multigrid.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_NLN_4x16_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_NLN_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_4x16_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_NLN_4x16_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_NLN_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_L.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_M.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_S.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_XS.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/C2D_NOPOOL_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/I3D_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/I3D_NLN_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_16x8_R101_50_50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_4x16_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R101_101_101.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R101_50_101.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R101_50_50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_NLN_16x8_R101_50_50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOW_4x16_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOW_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/C2D_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/CSN_32x2_R101.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/I3D_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/R2PLUS1D_16x4_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_16x8_R101_50_50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_4x16_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_8x8_R101.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOW_4x16_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOW_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_L.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_M.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_S.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_XS.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/SLOWFAST_16x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/SLOWFAST_16x8_R50_multigrid.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/pytorchvideo/SLOWFAST_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/pytorchvideo/SLOW_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/demo/AVA/Kinetics/SLOWFAST_8x8_R50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/demo/AVA/SLOWFAST_32x2_R101_50_50.yaml" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/demo/AVA/ava.json" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/__init__.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/config/__init__.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/config/custom_config.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/config/defaults.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/DATASET.md" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/__init__.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ava_dataset.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ava_helper.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/build.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/charades.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/cv2_transform.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/decoder.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/imagenet.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/kinetics.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/loader.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/mixup.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/multigrid_helper.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ptv_datasets.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/rand_augment.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/random_erasing.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ssv2.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/transform.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/utils.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/video_container.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/__init__.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/attention.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/batchnorm_helper.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/build.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/common.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/custom_video_model_builder.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/head_helper.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/losses.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/nonlocal_helper.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/operators.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/optimizer.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/ptv_model_builder.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/resnet_helper.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/stem_helper.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/utils.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/video_model_builder.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/__init__.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_eval_helper.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/README.md" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/__init__.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/ava_action_list_v2.1_for_activitynet_2018.pbtxt.txt" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/label_map_util.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/metrics.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_list.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_list_ops.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_mask_list.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_mask_list_ops.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_ops.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_mask_ops.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/object_detection_evaluation.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/per_image_evaluation.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/standard_fields.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/benchmark.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/bn_helper.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/c2_model_loading.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/checkpoint.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/distributed.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/env.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/logging.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/lr_policy.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/meters.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/metrics.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/misc.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/multigrid.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/multiprocessing.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/parser.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/weight_init_helper.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/__init__.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/async_predictor.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/ava_demo_precomputed_boxes.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/demo_loader.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/gradcam_utils.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/prediction_vis.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/predictor.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/tensorboard_vis.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/utils.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/video_visualizer.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/benchmark.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/demo_net.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/run_net.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/ss.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/test_net.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/train_net.py" create mode 100644 "code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/visualization.py" diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOWFAST_32x2_R50_SHORT.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOWFAST_32x2_R50_SHORT.yaml" new file mode 100644 index 0000000..ecaee7b --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOWFAST_32x2_R50_SHORT.yaml" @@ -0,0 +1,78 @@ +TRAIN: + ENABLE: True + DATASET: ava + BATCH_SIZE: 64 + EVAL_PERIOD: 5 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to the pretrain checkpoint file. + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3, 3] +DETECTION: + ENABLE: True + ALIGNED: True +AVA: + DETECTION_SCORE_THRESH: 0.8 + TRAIN_PREDICT_BOX_LISTS: [ + "ava_train_v2.2.csv", + "person_box_67091280_iou90/ava_detection_train_boxes_and_labels_include_negative_v2.2.csv", + ] + TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou90/ava_detection_val_boxes_and_labels.csv"] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [2, 2]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product + POOL: [[[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]]] +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: steps_with_relative_lrs + STEPS: [0, 10, 15, 20] + LRS: [1, 0.1, 0.01, 0.001] + MAX_EPOCH: 20 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-7 + WARMUP_EPOCHS: 5.0 + WARMUP_START_LR: 0.000125 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 80 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: bce + DROPOUT_RATE: 0.5 + HEAD_ACT: sigmoid +TEST: + ENABLE: True + DATASET: ava + BATCH_SIZE: 8 +DATA_LOADER: + NUM_WORKERS: 2 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOWFAST_32x2_R50_SHORT3.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOWFAST_32x2_R50_SHORT3.yaml" new file mode 100644 index 0000000..2c04d04 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOWFAST_32x2_R50_SHORT3.yaml" @@ -0,0 +1,91 @@ +TRAIN: + ENABLE: True + DATASET: ava + BATCH_SIZE: 2 #64 + EVAL_PERIOD: 5 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + CHECKPOINT_FILE_PATH: '/home/lxn/0yangfan/SlowFast/configs/AVA/c2/SLOWFAST_32x2_R101_50_50s.pkl' #path to pretrain model + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3, 3] + PATH_TO_DATA_DIR: '/disk6T/ava' +DETECTION: + ENABLE: True + ALIGNED: True +AVA: + FRAME_DIR: '/disk6T/ava/frames' + FRAME_LIST_DIR: '/disk6T/ava/frame_lists' + ANNOTATION_DIR: '/disk6T/ava/annotations' + #LABEL_MAP_FILE: 'ava_action_list_v2.1_for_activitynet_2018.pbtxt' + #0GROUNDTRUTH_FILE: 'ava_val_v2.1.csv' + #TRAIN_GT_BOX_LISTS: ['ava_train_v2.1.csv'] + DETECTION_SCORE_THRESH: 0.8 + TRAIN_PREDICT_BOX_LISTS: [ + "ava_train_v2.2.csv", + "person_box_67091280_iou90/ava_detection_train_boxes_and_labels_include_negative_v2.2.csv", + ] + #TRAIN_PREDICT_BOX_LISTS: ["ava_train_predicted_boxes.csv"] + TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou90/ava_detection_val_boxes_and_labels.csv"] + #TEST_PREDICT_BOX_LISTS: ["ava_test_predicted_boxes.csv"] + #EXCLUSION_FILE: "ava_train_excluded_timestamps_v2.1.csv" + + +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [2, 2]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product + POOL: [[[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]]] +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: steps_with_relative_lrs + STEPS: [0, 10, 15, 20] + LRS: [1, 0.1, 0.01, 0.001] + MAX_EPOCH: 20 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-7 + WARMUP_EPOCHS: 5.0 + WARMUP_START_LR: 0.000125 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 80 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: bce + DROPOUT_RATE: 0.5 + HEAD_ACT: sigmoid +TEST: + ENABLE: False + DATASET: ava + BATCH_SIZE: 8 +DATA_LOADER: + NUM_WORKERS: 2 + PIN_MEMORY: True +NUM_GPUS: 1 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . + diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOWFAST_32x2_R50_SHORT5.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOWFAST_32x2_R50_SHORT5.yaml" new file mode 100644 index 0000000..042ec50 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOWFAST_32x2_R50_SHORT5.yaml" @@ -0,0 +1,84 @@ +TRAIN: + ENABLE: True + DATASET: ava + BATCH_SIZE: 8 #64 + EVAL_PERIOD: 5 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + CHECKPOINT_FILE_PATH: '/home/lxn/0yangfan/Slowfast2/SlowFast-master/configs/AVA/c2/SLOWFAST_32x2_R101_50_50.pkl' #path to pretrain model + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3, 3] + PATH_TO_DATA_DIR: '/home/lxn/yf_videos/myCustomava' +DETECTION: + ENABLE: True + ALIGNED: True +AVA: + FRAME_DIR: '/home/lxn/yf_videos/myCustomava/frames' + FRAME_LIST_DIR: '/home/lxn/yf_videos/myCustomava/frame_lists' + ANNOTATION_DIR: '/home/lxn/yf_videos/myCustomava/annotations' + DETECTION_SCORE_THRESH: 0.8 + TRAIN_PREDICT_BOX_LISTS: [ + "ava_train_v2.2.csv", + "person_box_67091280_iou90/ava_detection_train_boxes_and_labels_include_negative_v2.2.csv", + ] + TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou90/ava_detection_val_boxes_and_labels.csv"] + + +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [2, 2]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product + POOL: [[[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]]] +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: steps_with_relative_lrs + STEPS: [0, 10, 15, 20] + LRS: [1, 0.1, 0.01, 0.001] + MAX_EPOCH: 20 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-7 + WARMUP_EPOCHS: 5.0 + WARMUP_START_LR: 0.000125 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 7 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: bce + DROPOUT_RATE: 0.5 + HEAD_ACT: sigmoid +TEST: + ENABLE: False + DATASET: ava + BATCH_SIZE: 8 +DATA_LOADER: + NUM_WORKERS: 2 + PIN_MEMORY: True +NUM_GPUS: 1 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOW_8x8_R50_SHORT.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOW_8x8_R50_SHORT.yaml" new file mode 100644 index 0000000..49ec7c3 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/SLOW_8x8_R50_SHORT.yaml" @@ -0,0 +1,72 @@ +TRAIN: + ENABLE: True + DATASET: ava + BATCH_SIZE: 64 + EVAL_PERIOD: 5 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to the pretrain checkpoint file. + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 4 + SAMPLING_RATE: 16 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +DETECTION: + ENABLE: True + ALIGNED: True +AVA: + DETECTION_SCORE_THRESH: 0.9 + TRAIN_PREDICT_BOX_LISTS: [ + "ava_train_v2.2.csv", + "person_box_67091280_iou90/ava_detection_train_boxes_and_labels_include_negative_v2.2.csv", + ] + TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou90/ava_detection_val_boxes_and_labels.csv"] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] + SPATIAL_DILATIONS: [[1], [1], [1], [2]] + SPATIAL_STRIDES: [[1], [2], [2], [1]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: steps_with_relative_lrs + STEPS: [0, 10, 15, 20] + LRS: [1, 0.1, 0.01, 0.001] + MAX_EPOCH: 20 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-7 + WARMUP_EPOCHS: 5.0 + WARMUP_START_LR: 0.000125 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 80 + ARCH: slow + MODEL_NAME: ResNet + LOSS_FUNC: bce + DROPOUT_RATE: 0.5 + HEAD_ACT: sigmoid +TEST: + ENABLE: True + DATASET: ava + BATCH_SIZE: 8 +DATA_LOADER: + NUM_WORKERS: 2 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_32x2_R101_50_50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_32x2_R101_50_50.yaml" new file mode 100644 index 0000000..481e059 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_32x2_R101_50_50.yaml" @@ -0,0 +1,68 @@ +TRAIN: + ENABLE: False + DATASET: ava + BATCH_SIZE: 16 + EVAL_PERIOD: 1 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to pretrain model + CHECKPOINT_TYPE: pytorch +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +DETECTION: + ENABLE: True + ALIGNED: False +AVA: + BGR: False + DETECTION_SCORE_THRESH: 0.8 + TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou90/ava_detection_val_boxes_and_labels.csv"] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 101 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [2, 2]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[6, 13, 20], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product + POOL: [[[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]]] +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-7 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 80 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: bce + DROPOUT_RATE: 0.5 + HEAD_ACT: sigmoid +TEST: + ENABLE: True + DATASET: ava + BATCH_SIZE: 8 +DATA_LOADER: + NUM_WORKERS: 2 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_32x2_R101_50_50_v2.1.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_32x2_R101_50_50_v2.1.yaml" new file mode 100644 index 0000000..e40d58e --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_32x2_R101_50_50_v2.1.yaml" @@ -0,0 +1,72 @@ +TRAIN: + ENABLE: False + DATASET: ava + BATCH_SIZE: 16 + EVAL_PERIOD: 1 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to pretrain model + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +DETECTION: + ENABLE: True + ALIGNED: False +AVA: + BGR: False + DETECTION_SCORE_THRESH: 0.8 + TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou90/ava_detection_val_boxes_and_labels.csv"] + TRAIN_GT_BOX_LISTS: ["ava_train_v2.1.csv"] + LABEL_MAP_FILE: ava_action_list_v2.1_for_activitynet_2018.pbtxt + EXCLUSION_FILE: ava_val_excluded_timestamps_v2.1.csv + GROUNDTRUTH_FILE: ava_val_v2.1.csv +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 101 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [2, 2]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[6, 13, 20], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product + POOL: [[[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]]] +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-7 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 80 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: bce + DROPOUT_RATE: 0.5 + HEAD_ACT: sigmoid +TEST: + ENABLE: True + DATASET: ava + BATCH_SIZE: 8 +DATA_LOADER: + NUM_WORKERS: 2 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_32x2_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_32x2_R50.yaml" new file mode 100644 index 0000000..d6f899b --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_32x2_R50.yaml" @@ -0,0 +1,68 @@ +TRAIN: + ENABLE: False + DATASET: ava + BATCH_SIZE: 16 + EVAL_PERIOD: 1 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to pretrain model + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +DETECTION: + ENABLE: True + ALIGNED: False +AVA: + BGR: False + DETECTION_SCORE_THRESH: 0.8 + TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou90/ava_detection_val_boxes_and_labels.csv"] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [2, 2]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product + POOL: [[[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]]] +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-7 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 80 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: bce + DROPOUT_RATE: 0.5 + HEAD_ACT: sigmoid +TEST: + ENABLE: True + DATASET: ava + BATCH_SIZE: 8 +DATA_LOADER: + NUM_WORKERS: 2 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_64x2_R101_50_50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_64x2_R101_50_50.yaml" new file mode 100644 index 0000000..2f85001 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOWFAST_64x2_R101_50_50.yaml" @@ -0,0 +1,68 @@ +TRAIN: + ENABLE: False + DATASET: ava + BATCH_SIZE: 16 + EVAL_PERIOD: 1 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to pretrain model + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 64 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +DETECTION: + ENABLE: True + ALIGNED: False +AVA: + BGR: False + DETECTION_SCORE_THRESH: 0.8 + TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou90/ava_detection_val_boxes_and_labels.csv"] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 101 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [2, 2]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[6, 13, 20], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product + POOL: [[[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]]] +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-7 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 80 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: bce + DROPOUT_RATE: 0.5 + HEAD_ACT: sigmoid +TEST: + ENABLE: True + DATASET: ava + BATCH_SIZE: 8 +DATA_LOADER: + NUM_WORKERS: 0 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOW_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOW_8x8_R50.yaml" new file mode 100644 index 0000000..23bffcf --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/AVA/c2/SLOW_8x8_R50.yaml" @@ -0,0 +1,62 @@ +TRAIN: + ENABLE: False + DATASET: ava + BATCH_SIZE: 64 + EVAL_PERIOD: 1 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to pretrain model + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 4 + SAMPLING_RATE: 16 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +DETECTION: + ENABLE: True + ALIGNED: False +AVA: + BGR: False + DETECTION_SCORE_THRESH: 0.9 + TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou75/ava_detection_val_boxes_and_labels.csv"] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] + SPATIAL_DILATIONS: [[1], [1], [1], [2]] + SPATIAL_STRIDES: [[1], [2], [2], [1]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-7 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 80 + ARCH: slow + MODEL_NAME: ResNet + LOSS_FUNC: bce + DROPOUT_RATE: 0.5 + HEAD_ACT: sigmoid +TEST: + ENABLE: True + DATASET: ava + BATCH_SIZE: 8 +DATA_LOADER: + NUM_WORKERS: 2 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/SLOWFAST_16x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/SLOWFAST_16x8_R50.yaml" new file mode 100644 index 0000000..b11b14a --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/SLOWFAST_16x8_R50.yaml" @@ -0,0 +1,75 @@ +TRAIN: + ENABLE: True + DATASET: charades + BATCH_SIZE: 16 + EVAL_PERIOD: 6 + CHECKPOINT_PERIOD: 6 + AUTO_RESUME: True + CHECKPOINT_FILE_PATH: SLOWFAST_8x8_R50.pkl # please download from the model zoo. + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 64 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 340] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] + MULTI_LABEL: True + INV_UNIFORM_SAMPLE: True + ENSEMBLE_METHOD: max + REVERSE_INPUT_CHANNEL: True +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 + NORM_TYPE: sync_batchnorm + NUM_SYNC_DEVICES: 4 +SOLVER: + BASE_LR: 0.0375 + LR_POLICY: steps_with_relative_lrs + LRS: [1, 0.1, 0.01, 0.001, 0.0001, 0.00001] + STEPS: [0, 41, 49] + MAX_EPOCH: 57 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 4.0 + WARMUP_START_LR: 0.0001 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 157 + ARCH: slowfast + LOSS_FUNC: bce_logit + HEAD_ACT: sigmoid + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: charades + BATCH_SIZE: 16 + NUM_ENSEMBLE_VIEWS: 10 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +LOG_MODEL_INFO: False diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/SLOWFAST_16x8_R50_multigrid.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/SLOWFAST_16x8_R50_multigrid.yaml" new file mode 100644 index 0000000..39dd864 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/SLOWFAST_16x8_R50_multigrid.yaml" @@ -0,0 +1,78 @@ +MULTIGRID: + SHORT_CYCLE: True + LONG_CYCLE: True +TRAIN: + ENABLE: True + DATASET: charades + BATCH_SIZE: 16 + EVAL_PERIOD: 6 + CHECKPOINT_PERIOD: 6 + AUTO_RESUME: True + CHECKPOINT_FILE_PATH: SLOWFAST_8x8_R50.pkl # please download from the model zoo. + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 64 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 340] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] + MULTI_LABEL: True + INV_UNIFORM_SAMPLE: True + ENSEMBLE_METHOD: max + REVERSE_INPUT_CHANNEL: True +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 + NORM_TYPE: sync_batchnorm + NUM_SYNC_DEVICES: 4 +SOLVER: + BASE_LR: 0.0375 + LR_POLICY: steps_with_relative_lrs + LRS: [1, 0.1, 0.01, 0.001, 0.0001, 0.00001] + STEPS: [0, 41, 49] + MAX_EPOCH: 57 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 4.0 + WARMUP_START_LR: 0.0001 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 157 + ARCH: slowfast + LOSS_FUNC: bce_logit + HEAD_ACT: sigmoid + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: charades + BATCH_SIZE: 16 + NUM_ENSEMBLE_VIEWS: 10 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +LOG_MODEL_INFO: False diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/pytorchvideo/SLOWFAST_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/pytorchvideo/SLOWFAST_8x8_R50.yaml" new file mode 100644 index 0000000..d290ff1 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/pytorchvideo/SLOWFAST_8x8_R50.yaml" @@ -0,0 +1,74 @@ +TRAIN: + ENABLE: True + DATASET: ptvcharades + BATCH_SIZE: 64 + EVAL_PERIOD: 6 + CHECKPOINT_PERIOD: 6 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: # please download from the model zoo. + CHECKPOINT_EPOCH_RESET: True +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] + MULTI_LABEL: True + INV_UNIFORM_SAMPLE: True + ENSEMBLE_METHOD: max + REVERSE_INPUT_CHANNEL: True +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product + POOL: [[[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]]] +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.25 + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + COSINE_END_LR: 1e-6 + MAX_EPOCH: 60 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 5.0 + WARMUP_START_LR: 0.0001 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 157 + ARCH: slowfast + MODEL_NAME: PTVSlowFast + LOSS_FUNC: bce_logit + HEAD_ACT: sigmoid + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ptvcharades + BATCH_SIZE: 64 + NUM_ENSEMBLE_VIEWS: 10 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 1 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/pytorchvideo/SLOW_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/pytorchvideo/SLOW_8x8_R50.yaml" new file mode 100644 index 0000000..9004adb --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Charades/pytorchvideo/SLOW_8x8_R50.yaml" @@ -0,0 +1,66 @@ +TRAIN: + ENABLE: True + DATASET: ptvcharades + BATCH_SIZE: 64 + EVAL_PERIOD: 6 + CHECKPOINT_PERIOD: 6 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: # please download from the model zoo. + CHECKPOINT_EPOCH_RESET: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 340] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] + MULTI_LABEL: True + INV_UNIFORM_SAMPLE: True + ENSEMBLE_METHOD: max + REVERSE_INPUT_CHANNEL: True +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.25 + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + COSINE_END_LR: 1e-6 + MAX_EPOCH: 60 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 5.0 + WARMUP_START_LR: 0.0001 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 157 + ARCH: slow + MODEL_NAME: PTVResNet + LOSS_FUNC: bce_logit + HEAD_ACT: sigmoid + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ptvcharades + BATCH_SIZE: 64 + NUM_ENSEMBLE_VIEWS: 10 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 1 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/ImageNet/MVIT_B_16_CONV.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/ImageNet/MVIT_B_16_CONV.yaml" new file mode 100644 index 0000000..adc6a4d --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/ImageNet/MVIT_B_16_CONV.yaml" @@ -0,0 +1,81 @@ +TRAIN: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 256 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + # PATH_TO_DATA_DIR: path-to-imagenet-dir + MEAN: [0.485, 0.456, 0.406] + STD: [0.229, 0.224, 0.225] + NUM_FRAMES: 1 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] +MVIT: + PATCH_2D: True + ZERO_DECAY_POS_CLS: False + MODE: "conv" + CLS_EMBED_ON: True + PATCH_KERNEL: [7, 7] + PATCH_STRIDE: [4, 4] + PATCH_PADDING: [3, 3] + EMBED_DIM: 96 + NUM_HEADS: 1 + MLP_RATIO: 4.0 + QKV_BIAS: True + DROPPATH_RATE: 0.1 + DEPTH: 16 + NORM: "layernorm" + DIM_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + HEAD_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + POOL_KVQ_KERNEL: [1, 3, 3] + POOL_KV_STRIDE: [[0, 1, 4, 4], [1, 1, 2, 2], [2, 1, 2, 2], [3, 1, 1, 1], [4, 1, 1, 1], [5, 1, 1, 1], [6, 1, 1, 1], [7, 1, 1, 1], [8, 1, 1, 1], [9, 1, 1, 1], [10, 1, 1, 1], [11, 1, 1, 1], [12, 1, 1, 1], [13, 1, 1, 1], [14, 1, 1, 1], [15, 1, 1, 1]] + POOL_Q_STRIDE: [[1, 1, 2, 2], [3, 1, 2, 2], [14, 1, 2, 2]] +AUG: + ENABLE: True + COLOR_JITTER: 0.4 + AA_TYPE: rand-m9-n6-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +SOLVER: + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.00025 + LR_POLICY: cosine + MAX_EPOCH: 300 + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + WARMUP_EPOCHS: 70.0 + WARMUP_START_LR: 1e-8 + OPTIMIZING_METHOD: adamw + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + ZERO_WD_1D_PARAM: True +MODEL: + NUM_CLASSES: 1000 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy + DROPOUT_RATE: 0.0 +TEST: + ENABLE: False + DATASET: imagenetprefetch + BATCH_SIZE: 256 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/ImageNet/RES_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/ImageNet/RES_R50.yaml" new file mode 100644 index 0000000..4916f45 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/ImageNet/RES_R50.yaml" @@ -0,0 +1,60 @@ +TRAIN: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 256 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + # PATH_TO_DATA_DIR: path-to-imagenet-dir + MEAN: [0.485, 0.456, 0.406] + STD: [0.229, 0.224, 0.225] + NUM_FRAMES: 1 + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] + TRAIN_JITTER_SCALES_RELATIVE: [0.08, 1.0] + TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333333333333333] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: steps_with_relative_lrs + MAX_EPOCH: 100 + STEPS: [0, 30, 60, 90] + LRS: [1, 0.1, 0.01, 0.001] + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 0.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 1000 + ARCH: 2d + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: imagenet + BATCH_SIZE: 256 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_8x8_R50.yaml" new file mode 100644 index 0000000..b0363d4 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_8x8_R50.yaml" @@ -0,0 +1,55 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: c2d + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_8x8_R50_IN1K.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_8x8_R50_IN1K.yaml" new file mode 100644 index 0000000..dfdc775 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_8x8_R50_IN1K.yaml" @@ -0,0 +1,58 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: /path/to/your/directory/imagenet50_pretrain.pyth + CHECKPOINT_INFLATE: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] + # PATH_TO_DATA_DIR: /path/to/your/dataset +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.01 + LR_POLICY: steps_with_relative_lrs + STEPS: [0, 44, 88, 118] + LRS: [1, 0.1, 0.01, 0.001] + MAX_EPOCH: 118 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: c2d + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_NLN_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_NLN_8x8_R50.yaml" new file mode 100644 index 0000000..86d60a1 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_NLN_8x8_R50.yaml" @@ -0,0 +1,55 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[1, 3]], [[1, 3, 5]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: c2d + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_NLN_8x8_R50_IN1K.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_NLN_8x8_R50_IN1K.yaml" new file mode 100644 index 0000000..2b5c411 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/C2D_NLN_8x8_R50_IN1K.yaml" @@ -0,0 +1,58 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: /path/to/your/directory/imagenet50_pretrain.pyth + CHECKPOINT_INFLATE: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] + # PATH_TO_DATA_DIR: /path/to/your/dataset +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[1, 3]], [[1, 3, 5]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.01 + LR_POLICY: steps_with_relative_lrs + STEPS: [0, 44, 88, 118] + LRS: [1, 0.1, 0.01, 0.001] + MAX_EPOCH: 118 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: c2d + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_8x8_R101.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_8x8_R101.yaml" new file mode 100644 index 0000000..b4e8418 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_8x8_R101.yaml" @@ -0,0 +1,55 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 101 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [23], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: i3d + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_8x8_R50.yaml" new file mode 100644 index 0000000..cb1b9e5 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_8x8_R50.yaml" @@ -0,0 +1,55 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: i3d + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_8x8_R50_IN1K.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_8x8_R50_IN1K.yaml" new file mode 100644 index 0000000..8ec682a --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_8x8_R50_IN1K.yaml" @@ -0,0 +1,58 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: /path/to/your/directory/imagenet50_pretrain.pyth + CHECKPOINT_INFLATE: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] + # PATH_TO_DATA_DIR: /path/to/your/dataset +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.01 + LR_POLICY: steps_with_relative_lrs + STEPS: [0, 44, 88, 118] + LRS: [1, 0.1, 0.01, 0.001] + MAX_EPOCH: 118 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: i3d + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_NLN_8x8_R101.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_NLN_8x8_R101.yaml" new file mode 100644 index 0000000..1bf0531 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_NLN_8x8_R101.yaml" @@ -0,0 +1,55 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 101 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [23], [3]] +NONLOCAL: + LOCATION: [[[]], [[1, 3]], [[1, 3, 5]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: i3d + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_NLN_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_NLN_8x8_R50.yaml" new file mode 100644 index 0000000..024c0ff --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_NLN_8x8_R50.yaml" @@ -0,0 +1,55 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[1, 3]], [[1, 3, 5]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: i3d + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_NLN_8x8_R50_IN1K.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_NLN_8x8_R50_IN1K.yaml" new file mode 100644 index 0000000..713b887 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/I3D_NLN_8x8_R50_IN1K.yaml" @@ -0,0 +1,58 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: /path/to/your/directory/imagenet50_pretrain.pyth + CHECKPOINT_INFLATE: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] + # PATH_TO_DATA_DIR: /path/to/your/dataset +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[1, 3]], [[1, 3, 5]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.01 + LR_POLICY: steps_with_relative_lrs + STEPS: [0, 44, 88, 118] + LRS: [1, 0.1, 0.01, 0.001] + MAX_EPOCH: 118 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: i3d + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/MVIT_B_16x4_CONV.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/MVIT_B_16x4_CONV.yaml" new file mode 100644 index 0000000..3d4363d --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/MVIT_B_16x4_CONV.yaml" @@ -0,0 +1,92 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 16 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True +DATA: + USE_OFFSET_SAMPLING: True + DECODING_BACKEND: torchvision + NUM_FRAMES: 16 + SAMPLING_RATE: 4 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3] + # PATH_TO_DATA_DIR: path-to-imagenet-dir + TRAIN_JITTER_SCALES_RELATIVE: [0.08, 1.0] + TRAIN_JITTER_ASPECT_RELATIVE: [0.75, 1.3333] +MVIT: + ZERO_DECAY_POS_CLS: False + SEP_POS_EMBED: True + DEPTH: 16 + NUM_HEADS: 1 + EMBED_DIM: 96 + PATCH_KERNEL: (3, 7, 7) + PATCH_STRIDE: (2, 4, 4) + PATCH_PADDING: (1, 3, 3) + MLP_RATIO: 4.0 + QKV_BIAS: True + DROPPATH_RATE: 0.2 + NORM: "layernorm" + MODE: "conv" + CLS_EMBED_ON: True + DIM_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + HEAD_MUL: [[1, 2.0], [3, 2.0], [14, 2.0]] + POOL_KVQ_KERNEL: [3, 3, 3] + POOL_KV_STRIDE: [[0, 1, 8, 8], [1, 1, 4, 4], [2, 1, 4, 4], [3, 1, 2, 2], [4, 1, 2, 2], [5, 1, 2, 2], [6, 1, 2, 2], [7, 1, 2, 2], [8, 1, 2, 2], [9, 1, 2, 2], [10, 1, 2, 2], [11, 1, 2, 2], [12, 1, 2, 2], [13, 1, 2, 2]] + POOL_Q_STRIDE: [[1, 1, 2, 2], [3, 1, 2, 2], [14, 1, 2, 2]] + DROPOUT_RATE: 0.0 +AUG: + NUM_SAMPLE: 2 + ENABLE: True + COLOR_JITTER: 0.4 + AA_TYPE: rand-m7-n4-mstd0.5-inc1 + INTERPOLATION: bicubic + RE_PROB: 0.25 + RE_MODE: pixel + RE_COUNT: 1 + RE_SPLIT: False +MIXUP: + ENABLE: True + ALPHA: 0.8 + CUTMIX_ALPHA: 1.0 + PROB: 1.0 + SWITCH_PROB: 0.5 + LABEL_SMOOTH_VALUE: 0.1 +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + ZERO_WD_1D_PARAM: True + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.0001 + COSINE_AFTER_WARMUP: True + COSINE_END_LR: 1e-6 + WARMUP_START_LR: 1e-6 + WARMUP_EPOCHS: 30.0 + LR_POLICY: cosine + MAX_EPOCH: 200 + MOMENTUM: 0.9 + WEIGHT_DECAY: 0.05 + OPTIMIZING_METHOD: adamw + COSINE_AFTER_WARMUP: True +MODEL: + NUM_CLASSES: 400 + ARCH: mvit + MODEL_NAME: MViT + LOSS_FUNC: soft_cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 1 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_4x16_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_4x16_R50.yaml" new file mode 100644 index 0000000..8398e2b --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_4x16_R50.yaml" @@ -0,0 +1,62 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +SLOWFAST: + ALPHA: 8 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_8x8_R50.yaml" new file mode 100644 index 0000000..0ba7144 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_8x8_R50.yaml" @@ -0,0 +1,62 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_8x8_R50_stepwise.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_8x8_R50_stepwise.yaml" new file mode 100644 index 0000000..6d64185 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_8x8_R50_stepwise.yaml" @@ -0,0 +1,64 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: steps_with_relative_lrs + LRS: [1, 0.1, 0.01, 0.001, 0.0001, 0.00001] + STEPS: [0, 94, 154, 196] + MAX_EPOCH: 239 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_8x8_R50_stepwise_multigrid.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_8x8_R50_stepwise_multigrid.yaml" new file mode 100644 index 0000000..63a3c68 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_8x8_R50_stepwise_multigrid.yaml" @@ -0,0 +1,67 @@ +MULTIGRID: + SHORT_CYCLE: True + LONG_CYCLE: True +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3, 3] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: steps_with_relative_lrs + LRS: [1, 0.1, 0.01, 0.001, 0.0001, 0.00001] + STEPS: [0, 94, 154, 196] + MAX_EPOCH: 239 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_NLN_4x16_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_NLN_4x16_R50.yaml" new file mode 100644 index 0000000..d1045e5 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_NLN_4x16_R50.yaml" @@ -0,0 +1,62 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +SLOWFAST: + ALPHA: 8 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[1, 3], []], [[1, 3, 5], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_NLN_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_NLN_8x8_R50.yaml" new file mode 100644 index 0000000..621026d --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOWFAST_NLN_8x8_R50.yaml" @@ -0,0 +1,62 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[1, 3], []], [[1, 3, 5], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_4x16_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_4x16_R50.yaml" new file mode 100644 index 0000000..c37125d --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_4x16_R50.yaml" @@ -0,0 +1,55 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 4 + SAMPLING_RATE: 16 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slow + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_8x8_R50.yaml" new file mode 100644 index 0000000..1faf5f9 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_8x8_R50.yaml" @@ -0,0 +1,55 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slow + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_NLN_4x16_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_NLN_4x16_R50.yaml" new file mode 100644 index 0000000..1387c48 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_NLN_4x16_R50.yaml" @@ -0,0 +1,55 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 4 + SAMPLING_RATE: 16 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[1, 3]], [[1, 3, 5]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slow + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_NLN_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_NLN_8x8_R50.yaml" new file mode 100644 index 0000000..71f37da --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/SLOW_NLN_8x8_R50.yaml" @@ -0,0 +1,55 @@ +TRAIN: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[1, 3]], [[1, 3, 5]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slow + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_L.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_L.yaml" new file mode 100644 index 0000000..7cfe32c --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_L.yaml" @@ -0,0 +1,58 @@ +TRAIN: + # ENABLE: False # default True + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True +X3D: + WIDTH_FACTOR: 2.0 + DEPTH_FACTOR: 5.0 + BOTTLENECK_FACTOR: 2.25 + DIM_C5: 2048 + DIM_C1: 12 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 32 + # CHECKPOINT_FILE_PATH: 'x3d_l.pyth' # 77.48% top1 30-view accuracy to download from the model zoo (optional). + # NUM_SPATIAL_CROPS: 1 + NUM_SPATIAL_CROPS: 3 +DATA: + NUM_FRAMES: 16 + SAMPLING_RATE: 5 + TRAIN_JITTER_SCALES: [356, 446] + TRAIN_CROP_SIZE: 312 + # TEST_CROP_SIZE: 312 # use if TEST.NUM_SPATIAL_CROPS: 1 + TEST_CROP_SIZE: 356 # use if TEST.NUM_SPATIAL_CROPS: 3 + INPUT_CHANNEL_NUM: [3] + DECODING_BACKEND: torchvision +RESNET: + ZERO_INIT_FINAL_BN: True + TRANS_FUNC: x3d_transform + STRIDE_1X1: False +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 + WEIGHT_DECAY: 0.0 +SOLVER: + BASE_LR: 0.05 # 1 machine + BASE_LR_SCALE_NUM_SHARDS: True + MAX_EPOCH: 256 + LR_POLICY: cosine + WEIGHT_DECAY: 5e-5 + WARMUP_EPOCHS: 35.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: x3d + MODEL_NAME: X3D + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_M.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_M.yaml" new file mode 100644 index 0000000..b99f7cb --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_M.yaml" @@ -0,0 +1,58 @@ +TRAIN: + # ENABLE: False # default True + DATASET: kinetics + BATCH_SIZE: 128 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True +X3D: + WIDTH_FACTOR: 2.0 + DEPTH_FACTOR: 2.2 + BOTTLENECK_FACTOR: 2.25 + DIM_C5: 2048 + DIM_C1: 12 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + # CHECKPOINT_FILE_PATH: 'x3d_m.pyth' # 76.21% top1 30-view accuracy to download from the model zoo (optional). + # NUM_SPATIAL_CROPS: 1 + NUM_SPATIAL_CROPS: 3 +DATA: + NUM_FRAMES: 16 + SAMPLING_RATE: 5 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + # TEST_CROP_SIZE: 224 # use if TEST.NUM_SPATIAL_CROPS: 1 + TEST_CROP_SIZE: 256 # use if TEST.NUM_SPATIAL_CROPS: 3 + INPUT_CHANNEL_NUM: [3] + DECODING_BACKEND: torchvision +RESNET: + ZERO_INIT_FINAL_BN: True + TRANS_FUNC: x3d_transform + STRIDE_1X1: False +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 + WEIGHT_DECAY: 0.0 +SOLVER: + BASE_LR: 0.1 # 1 machine + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 300 + WEIGHT_DECAY: 5e-5 + WARMUP_EPOCHS: 35.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: x3d + MODEL_NAME: X3D + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_S.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_S.yaml" new file mode 100644 index 0000000..33dd5f7 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_S.yaml" @@ -0,0 +1,58 @@ +TRAIN: + # ENABLE: False # default True + DATASET: kinetics + BATCH_SIZE: 128 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True +X3D: + WIDTH_FACTOR: 2.0 + DEPTH_FACTOR: 2.2 + BOTTLENECK_FACTOR: 2.25 + DIM_C5: 2048 + DIM_C1: 12 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + # CHECKPOINT_FILE_PATH: 'x3d_s.pyth' # 73.50% top1 30-view accuracy to download from the model zoo (optional). + # NUM_SPATIAL_CROPS: 1 + NUM_SPATIAL_CROPS: 3 +DATA: + NUM_FRAMES: 13 + SAMPLING_RATE: 6 + TRAIN_JITTER_SCALES: [182, 228] + TRAIN_CROP_SIZE: 160 + # TEST_CROP_SIZE: 160 # use if TEST.NUM_SPATIAL_CROPS: 1 + TEST_CROP_SIZE: 182 # use if TEST.NUM_SPATIAL_CROPS: 3 + INPUT_CHANNEL_NUM: [3] + DECODING_BACKEND: torchvision +RESNET: + ZERO_INIT_FINAL_BN: True + TRANS_FUNC: x3d_transform + STRIDE_1X1: False +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 + WEIGHT_DECAY: 0.0 +SOLVER: + BASE_LR: 0.1 # 16 machine + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 300 + WEIGHT_DECAY: 5e-5 + WARMUP_EPOCHS: 35.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: x3d + MODEL_NAME: X3D + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_XS.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_XS.yaml" new file mode 100644 index 0000000..e02a441 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/X3D_XS.yaml" @@ -0,0 +1,58 @@ +TRAIN: + # ENABLE: False # default True + DATASET: kinetics + BATCH_SIZE: 128 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True +X3D: + WIDTH_FACTOR: 2.0 + DEPTH_FACTOR: 2.2 + BOTTLENECK_FACTOR: 2.25 + DIM_C5: 2048 + DIM_C1: 12 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 + # CHECKPOINT_FILE_PATH: 'x3d_xs.pyth' # 69.46% top1 30-view accuracy to download from the model zoo (optional). + # NUM_SPATIAL_CROPS: 1 + NUM_SPATIAL_CROPS: 3 +DATA: + NUM_FRAMES: 4 + SAMPLING_RATE: 12 + TRAIN_JITTER_SCALES: [182, 228] + TRAIN_CROP_SIZE: 160 + # TEST_CROP_SIZE: 160 # use if TEST.NUM_SPATIAL_CROPS: 1 + TEST_CROP_SIZE: 182 # use if TEST.NUM_SPATIAL_CROPS: 3 + INPUT_CHANNEL_NUM: [3] + DECODING_BACKEND: torchvision +RESNET: + ZERO_INIT_FINAL_BN: True + TRANS_FUNC: x3d_transform + STRIDE_1X1: False +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 + WEIGHT_DECAY: 0.0 +SOLVER: + BASE_LR: 0.1 # 1 machine + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 300 + WEIGHT_DECAY: 5e-5 + WARMUP_EPOCHS: 35.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: x3d + MODEL_NAME: X3D + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/C2D_NOPOOL_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/C2D_NOPOOL_8x8_R50.yaml" new file mode 100644 index 0000000..f8bb2d9 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/C2D_NOPOOL_8x8_R50.yaml" @@ -0,0 +1,57 @@ +TRAIN: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to the model to test + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: c2d + MODEL_NAME: ResNet_nopool + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/I3D_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/I3D_8x8_R50.yaml" new file mode 100644 index 0000000..f0e4e4f --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/I3D_8x8_R50.yaml" @@ -0,0 +1,57 @@ +TRAIN: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to the model to test + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: i3d + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/I3D_NLN_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/I3D_NLN_8x8_R50.yaml" new file mode 100644 index 0000000..c2b4153 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/I3D_NLN_8x8_R50.yaml" @@ -0,0 +1,57 @@ +TRAIN: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to the model to test + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[1, 3]], [[1, 3, 5]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: softmax +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: i3d + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_16x8_R101_50_50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_16x8_R101_50_50.yaml" new file mode 100644 index 0000000..1d0d2fc --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_16x8_R101_50_50.yaml" @@ -0,0 +1,64 @@ +TRAIN: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to the model to test + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 64 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 101 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_4x16_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_4x16_R50.yaml" new file mode 100644 index 0000000..4f74227 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_4x16_R50.yaml" @@ -0,0 +1,64 @@ +TRAIN: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to the model to test + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +SLOWFAST: + ALPHA: 8 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R101_101_101.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R101_101_101.yaml" new file mode 100644 index 0000000..28b5b7b --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R101_101_101.yaml" @@ -0,0 +1,64 @@ +TRAIN: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to the model to test + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 101 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [23, 23], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R101_50_101.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R101_50_101.yaml" new file mode 100644 index 0000000..c3df8a8 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R101_50_101.yaml" @@ -0,0 +1,64 @@ +TRAIN: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to the model to test + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 101 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 23], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R101_50_50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R101_50_50.yaml" new file mode 100644 index 0000000..76451f2 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R101_50_50.yaml" @@ -0,0 +1,64 @@ +TRAIN: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to the model to test + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 101 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R50.yaml" new file mode 100644 index 0000000..d6f0a97 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_8x8_R50.yaml" @@ -0,0 +1,64 @@ +TRAIN: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to the model to test + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_NLN_16x8_R101_50_50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_NLN_16x8_R101_50_50.yaml" new file mode 100644 index 0000000..3f8c7eb --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOWFAST_NLN_16x8_R101_50_50.yaml" @@ -0,0 +1,64 @@ +TRAIN: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to the model to test + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 64 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 101 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[6, 13, 20], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOW_4x16_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOW_4x16_R50.yaml" new file mode 100644 index 0000000..6c95688 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOW_4x16_R50.yaml" @@ -0,0 +1,57 @@ +TRAIN: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to the model to test + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 4 + SAMPLING_RATE: 16 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slow + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOW_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOW_8x8_R50.yaml" new file mode 100644 index 0000000..44cc47e --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/c2/SLOW_8x8_R50.yaml" @@ -0,0 +1,57 @@ +TRAIN: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: path to the model to test + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slow + MODEL_NAME: ResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/C2D_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/C2D_8x8_R50.yaml" new file mode 100644 index 0000000..374b7c5 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/C2D_8x8_R50.yaml" @@ -0,0 +1,59 @@ +TRAIN: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 # 1-node + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: c2d + MODEL_NAME: PTVResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +TENSORBOARD: + ENABLE: True diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/CSN_32x2_R101.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/CSN_32x2_R101.yaml" new file mode 100644 index 0000000..b735fcc --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/CSN_32x2_R101.yaml" @@ -0,0 +1,58 @@ +TRAIN: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] + + +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + DEPTH: 101 +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 # 1-node + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: csn + MODEL_NAME: PTVCSN + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 + SINGLE_PATHWAY_ARCH: ['csn'] +TEST: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +TENSORBOARD: + ENABLE: True diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/I3D_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/I3D_8x8_R50.yaml" new file mode 100644 index 0000000..a49b540 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/I3D_8x8_R50.yaml" @@ -0,0 +1,61 @@ +TRAIN: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] + + +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 # 1-node + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: i3d + MODEL_NAME: PTVResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +TENSORBOARD: + ENABLE: True diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/R2PLUS1D_16x4_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/R2PLUS1D_16x4_R50.yaml" new file mode 100644 index 0000000..8c5d476 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/R2PLUS1D_16x4_R50.yaml" @@ -0,0 +1,58 @@ +TRAIN: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 16 + SAMPLING_RATE: 4 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] + + +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + DEPTH: 50 +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 # 1-node + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: r2plus1d + MODEL_NAME: PTVR2plus1D + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 + SINGLE_PATHWAY_ARCH: ['r2plus1d'] +TEST: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +TENSORBOARD: + ENABLE: True diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_16x8_R101_50_50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_16x8_R101_50_50.yaml" new file mode 100644 index 0000000..6fb8b66 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_16x8_R101_50_50.yaml" @@ -0,0 +1,66 @@ +TRAIN: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 2 + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 64 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 360] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 101 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 # 1-node + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: PTVSlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +TENSORBOARD: + ENABLE: True diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_4x16_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_4x16_R50.yaml" new file mode 100644 index 0000000..2ffa393 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_4x16_R50.yaml" @@ -0,0 +1,68 @@ +TRAIN: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] + + +SLOWFAST: + ALPHA: 8 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 # 1-node + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: PTVSlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +TENSORBOARD: + ENABLE: True diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_8x8_R101.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_8x8_R101.yaml" new file mode 100644 index 0000000..fc29abd --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_8x8_R101.yaml" @@ -0,0 +1,68 @@ +TRAIN: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 2 + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] + + +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 101 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [23, 23], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 # 1-node + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: PTVSlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +TENSORBOARD: + ENABLE: True diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_8x8_R50.yaml" new file mode 100644 index 0000000..1af2c7b --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOWFAST_8x8_R50.yaml" @@ -0,0 +1,68 @@ +TRAIN: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] + + +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 # 1-node + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + MODEL_NAME: PTVSlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +TENSORBOARD: + ENABLE: True diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOW_4x16_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOW_4x16_R50.yaml" new file mode 100644 index 0000000..19e4b8d --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOW_4x16_R50.yaml" @@ -0,0 +1,61 @@ +TRAIN: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 4 + SAMPLING_RATE: 16 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] + + +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 # 1-node + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slow + MODEL_NAME: PTVResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +TENSORBOARD: + ENABLE: True diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOW_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOW_8x8_R50.yaml" new file mode 100644 index 0000000..81360fd --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/SLOW_8x8_R50.yaml" @@ -0,0 +1,61 @@ +TRAIN: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] + + +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 # 1-node + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slow + MODEL_NAME: PTVResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +TENSORBOARD: + ENABLE: True diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_L.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_L.yaml" new file mode 100644 index 0000000..de44aba --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_L.yaml" @@ -0,0 +1,55 @@ +TRAIN: + DATASET: ptvkinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 2 + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True +X3D: + WIDTH_FACTOR: 2.0 + DEPTH_FACTOR: 5.0 + BOTTLENECK_FACTOR: 2.25 + DIM_C5: 2048 + DIM_C1: 12 +TEST: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 32 + # NUM_SPATIAL_CROPS: 1 + NUM_SPATIAL_CROPS: 3 +DATA: + NUM_FRAMES: 16 + SAMPLING_RATE: 5 + TRAIN_JITTER_SCALES: [356, 446] + TRAIN_CROP_SIZE: 312 + # TEST_CROP_SIZE: 312 # use if TEST.NUM_SPATIAL_CROPS: 1 + TEST_CROP_SIZE: 356 # use if TEST.NUM_SPATIAL_CROPS: 3 + INPUT_CHANNEL_NUM: [3] +RESNET: + ZERO_INIT_FINAL_BN: True + TRANS_FUNC: x3d_transform + STRIDE_1X1: False +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 + WEIGHT_DECAY: 0.0 +SOLVER: + BASE_LR: 0.05 # 1 machine + BASE_LR_SCALE_NUM_SHARDS: True + MAX_EPOCH: 256 + LR_POLICY: cosine + WEIGHT_DECAY: 5e-5 + WARMUP_EPOCHS: 35.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: x3d + MODEL_NAME: PTVX3D + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_M.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_M.yaml" new file mode 100644 index 0000000..85ce8c5 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_M.yaml" @@ -0,0 +1,59 @@ +TRAIN: + # ENABLE: False # default True + DATASET: ptvkinetics + BATCH_SIZE: 128 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True +X3D: + WIDTH_FACTOR: 2.0 + DEPTH_FACTOR: 2.2 + BOTTLENECK_FACTOR: 2.25 + DIM_C5: 2048 + DIM_C1: 12 +TEST: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + # CHECKPOINT_FILE_PATH: 'x3d_m.pyth' # 76.21% top1 30-view accuracy to download from the model zoo (optional). + # NUM_SPATIAL_CROPS: 1 + NUM_SPATIAL_CROPS: 3 +DATA: + NUM_FRAMES: 16 + SAMPLING_RATE: 5 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + # TEST_CROP_SIZE: 224 # use if TEST.NUM_SPATIAL_CROPS: 1 + TEST_CROP_SIZE: 256 # use if TEST.NUM_SPATIAL_CROPS: 3 + INPUT_CHANNEL_NUM: [3] + + +RESNET: + ZERO_INIT_FINAL_BN: True + TRANS_FUNC: x3d_transform + STRIDE_1X1: False +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 + WEIGHT_DECAY: 0.0 +SOLVER: + BASE_LR: 0.1 # 1 machine + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 300 + WEIGHT_DECAY: 5e-5 + WARMUP_EPOCHS: 35.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: x3d + MODEL_NAME: PTVX3D + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_S.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_S.yaml" new file mode 100644 index 0000000..2dba575 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_S.yaml" @@ -0,0 +1,60 @@ +TRAIN: + # ENABLE: False # default True + DATASET: ptvkinetics + BATCH_SIZE: 128 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True +X3D: + WIDTH_FACTOR: 2.0 + DEPTH_FACTOR: 2.2 + BOTTLENECK_FACTOR: 2.25 + DIM_C5: 2048 + DIM_C1: 12 +TEST: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + # CHECKPOINT_FILE_PATH: 'x3d_s.pyth' # 73.50% top1 30-view accuracy to download from the model zoo (optional). + # NUM_SPATIAL_CROPS: 1 + NUM_SPATIAL_CROPS: 3 +DATA: + NUM_FRAMES: 13 + SAMPLING_RATE: 6 + TRAIN_JITTER_SCALES: [182, 228] + TRAIN_CROP_SIZE: 160 + # TEST_CROP_SIZE: 160 # use if TEST.NUM_SPATIAL_CROPS: 1 + TEST_CROP_SIZE: 182 # use if TEST.NUM_SPATIAL_CROPS: 3 + INPUT_CHANNEL_NUM: [3] + # DECODING_BACKEND: torchvision + + +RESNET: + ZERO_INIT_FINAL_BN: True + TRANS_FUNC: x3d_transform + STRIDE_1X1: False +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 + WEIGHT_DECAY: 0.0 +SOLVER: + BASE_LR: 0.1 # 16 machine + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 300 + WEIGHT_DECAY: 5e-5 + WARMUP_EPOCHS: 35.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: x3d + MODEL_NAME: PTVX3D + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_XS.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_XS.yaml" new file mode 100644 index 0000000..5ac31a3 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/Kinetics/pytorchvideo/X3D_XS.yaml" @@ -0,0 +1,58 @@ +TRAIN: + # ENABLE: False # default True + DATASET: ptvkinetics + BATCH_SIZE: 128 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 10 + AUTO_RESUME: True +X3D: + WIDTH_FACTOR: 2.0 + DEPTH_FACTOR: 2.2 + BOTTLENECK_FACTOR: 2.25 + DIM_C5: 2048 + DIM_C1: 12 +TEST: + ENABLE: True + DATASET: ptvkinetics + BATCH_SIZE: 64 + # CHECKPOINT_FILE_PATH: 'x3d_xs.pyth' # 69.46% top1 30-view accuracy to download from the model zoo (optional). + # NUM_SPATIAL_CROPS: 1 + NUM_SPATIAL_CROPS: 3 +DATA: + NUM_FRAMES: 4 + SAMPLING_RATE: 12 + TRAIN_JITTER_SCALES: [182, 228] + TRAIN_CROP_SIZE: 160 + # TEST_CROP_SIZE: 160 # use if TEST.NUM_SPATIAL_CROPS: 1 + TEST_CROP_SIZE: 182 # use if TEST.NUM_SPATIAL_CROPS: 3 + INPUT_CHANNEL_NUM: [3] + +RESNET: + ZERO_INIT_FINAL_BN: True + TRANS_FUNC: x3d_transform + STRIDE_1X1: False +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 + WEIGHT_DECAY: 0.0 +SOLVER: + BASE_LR: 0.1 # 1 machine + BASE_LR_SCALE_NUM_SHARDS: True + LR_POLICY: cosine + MAX_EPOCH: 300 + WEIGHT_DECAY: 5e-5 + WARMUP_EPOCHS: 35.0 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: x3d + MODEL_NAME: PTVX3D + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +NUM_GPUS: 8 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/SLOWFAST_16x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/SLOWFAST_16x8_R50.yaml" new file mode 100644 index 0000000..93939e3 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/SLOWFAST_16x8_R50.yaml" @@ -0,0 +1,73 @@ +TRAIN: + ENABLE: True + DATASET: ssv2 + BATCH_SIZE: 16 + EVAL_PERIOD: 2 + CHECKPOINT_PERIOD: 2 + AUTO_RESUME: True + CHECKPOINT_FILE_PATH: SLOWFAST_8x8_R50.pkl # please download from the model zoo. + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 64 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] + INV_UNIFORM_SAMPLE: True + RANDOM_FLIP: False + REVERSE_INPUT_CHANNEL: True +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 + NORM_TYPE: sync_batchnorm + NUM_SYNC_DEVICES: 4 +SOLVER: + BASE_LR: 0.03 + LR_POLICY: steps_with_relative_lrs + LRS: [1, 0.1, 0.01, 0.001, 0.0001, 0.00001] + STEPS: [0, 14, 18] + MAX_EPOCH: 22 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-6 + WARMUP_EPOCHS: 0.19 + WARMUP_START_LR: 0.0001 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 174 + ARCH: slowfast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ssv2 + BATCH_SIZE: 16 + NUM_ENSEMBLE_VIEWS: 1 + NUM_SPATIAL_CROPS: 1 +DATA_LOADER: + NUM_WORKERS: 4 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +LOG_MODEL_INFO: False diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/SLOWFAST_16x8_R50_multigrid.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/SLOWFAST_16x8_R50_multigrid.yaml" new file mode 100644 index 0000000..5c6e3a2 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/SLOWFAST_16x8_R50_multigrid.yaml" @@ -0,0 +1,76 @@ +MULTIGRID: + SHORT_CYCLE: True + LONG_CYCLE: True +TRAIN: + ENABLE: True + DATASET: ssv2 + BATCH_SIZE: 16 + EVAL_PERIOD: 2 + CHECKPOINT_PERIOD: 2 + AUTO_RESUME: True + CHECKPOINT_FILE_PATH: SLOWFAST_8x8_R50.pkl # please download from the model zoo. + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 64 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 224 + INPUT_CHANNEL_NUM: [3, 3] + INV_UNIFORM_SAMPLE: True + RANDOM_FLIP: False + REVERSE_INPUT_CHANNEL: True +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 + NORM_TYPE: sync_batchnorm + NUM_SYNC_DEVICES: 4 +SOLVER: + BASE_LR: 0.03 + LR_POLICY: steps_with_relative_lrs + LRS: [1, 0.1, 0.01, 0.001, 0.0001, 0.00001] + STEPS: [0, 14, 18] + MAX_EPOCH: 22 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-6 + WARMUP_EPOCHS: 0.19 + WARMUP_START_LR: 0.0001 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 174 + ARCH: slowfast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ssv2 + BATCH_SIZE: 16 + NUM_ENSEMBLE_VIEWS: 1 + NUM_SPATIAL_CROPS: 1 +DATA_LOADER: + NUM_WORKERS: 4 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +LOG_MODEL_INFO: False diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/pytorchvideo/SLOWFAST_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/pytorchvideo/SLOWFAST_8x8_R50.yaml" new file mode 100644 index 0000000..13437b2 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/pytorchvideo/SLOWFAST_8x8_R50.yaml" @@ -0,0 +1,71 @@ +TRAIN: + ENABLE: True + DATASET: ptvssv2 + BATCH_SIZE: 64 + EVAL_PERIOD: 2 + CHECKPOINT_PERIOD: 2 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: # please download from the model zoo. + CHECKPOINT_EPOCH_RESET: True +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] + INV_UNIFORM_SAMPLE: True + RANDOM_FLIP: False + REVERSE_INPUT_CHANNEL: True +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product + POOL: [[[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]], [[1, 2, 2], [1, 2, 2]]] +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 150 +SOLVER: + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 30 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 3.0 + WARMUP_START_LR: 0.08 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 174 + ARCH: slowfast + MODEL_NAME: PTVSlowFast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ptvssv2 + BATCH_SIZE: 64 + NUM_ENSEMBLE_VIEWS: 1 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 2 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/pytorchvideo/SLOW_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/pytorchvideo/SLOW_8x8_R50.yaml" new file mode 100644 index 0000000..9efa213 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/configs/SSv2/pytorchvideo/SLOW_8x8_R50.yaml" @@ -0,0 +1,63 @@ +TRAIN: + ENABLE: True + DATASET: ptvssv2 + BATCH_SIZE: 64 + EVAL_PERIOD: 2 + CHECKPOINT_PERIOD: 2 + AUTO_RESUME: True + # CHECKPOINT_FILE_PATH: # please download from the model zoo. + CHECKPOINT_EPOCH_RESET: True +DATA: + NUM_FRAMES: 8 + SAMPLING_RATE: 8 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3] + INV_UNIFORM_SAMPLE: True + RANDOM_FLIP: False + REVERSE_INPUT_CHANNEL: True +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3], [4], [6], [3]] +NONLOCAL: + LOCATION: [[[]], [[]], [[]], [[]]] + GROUP: [[1], [1], [1], [1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR_SCALE_NUM_SHARDS: True + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 30 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 3.0 + WARMUP_START_LR: 0.08 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 174 + ARCH: slow + MODEL_NAME: PTVResNet + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: True + DATASET: ptvssv2 + BATCH_SIZE: 64 + NUM_ENSEMBLE_VIEWS: 1 + NUM_SPATIAL_CROPS: 3 +DATA_LOADER: + NUM_WORKERS: 2 + PIN_MEMORY: True +NUM_GPUS: 8 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/demo/AVA/Kinetics/SLOWFAST_8x8_R50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/demo/AVA/Kinetics/SLOWFAST_8x8_R50.yaml" new file mode 100644 index 0000000..baead7b --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/demo/AVA/Kinetics/SLOWFAST_8x8_R50.yaml" @@ -0,0 +1,67 @@ +TRAIN: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 + EVAL_PERIOD: 10 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + CHECKPOINT_FILE_PATH: "./SLOWFAST_8x8_R50.pkl" # path to pretrain model to run demo + CHECKPOINT_TYPE: caffe2 +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 7 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 50 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [2, 2]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product +BN: + USE_PRECISE_STATS: True + NUM_BATCHES_PRECISE: 200 +SOLVER: + BASE_LR: 0.1 + LR_POLICY: cosine + MAX_EPOCH: 196 + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-4 + WARMUP_EPOCHS: 34 + WARMUP_START_LR: 0.01 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 400 + ARCH: slowfast + LOSS_FUNC: cross_entropy + DROPOUT_RATE: 0.5 +TEST: + ENABLE: False + DATASET: kinetics + BATCH_SIZE: 64 +DATA_LOADER: + NUM_WORKERS: 8 + PIN_MEMORY: True +DEMO: + ENABLE: True + LABEL_FILE_PATH: # Add local label file path here. + WEBCAM: 0 +NUM_GPUS: 1 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/demo/AVA/SLOWFAST_32x2_R101_50_50.yaml" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/demo/AVA/SLOWFAST_32x2_R101_50_50.yaml" new file mode 100644 index 0000000..8e5e494 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/demo/AVA/SLOWFAST_32x2_R101_50_50.yaml" @@ -0,0 +1,80 @@ +TRAIN: + ENABLE: False + DATASET: ava + BATCH_SIZE: 16 + EVAL_PERIOD: 1 + CHECKPOINT_PERIOD: 1 + AUTO_RESUME: True + CHECKPOINT_FILE_PATH: '/home/SlowFast-master/configs/AVA/c2/SLOWFAST_32x2_R101_50_50.pkl' #path to pretrain model + CHECKPOINT_TYPE: pytorch +DATA: + NUM_FRAMES: 32 + SAMPLING_RATE: 2 + TRAIN_JITTER_SCALES: [256, 320] + TRAIN_CROP_SIZE: 224 + TEST_CROP_SIZE: 256 + INPUT_CHANNEL_NUM: [3, 3] +DETECTION: + ENABLE: True + ALIGNED: False +AVA: + BGR: False + DETECTION_SCORE_THRESH: 0.8 + TEST_PREDICT_BOX_LISTS: ["person_box_67091280_iou90/ava_detection_val_boxes_and_labels.csv"] +SLOWFAST: + ALPHA: 4 + BETA_INV: 8 + FUSION_CONV_CHANNEL_RATIO: 2 + FUSION_KERNEL_SZ: 5 +RESNET: + ZERO_INIT_FINAL_BN: True + WIDTH_PER_GROUP: 64 + NUM_GROUPS: 1 + DEPTH: 101 + TRANS_FUNC: bottleneck_transform + STRIDE_1X1: False + NUM_BLOCK_TEMP_KERNEL: [[3, 3], [4, 4], [6, 6], [3, 3]] + SPATIAL_DILATIONS: [[1, 1], [1, 1], [1, 1], [2, 2]] + SPATIAL_STRIDES: [[1, 1], [2, 2], [2, 2], [1, 1]] +NONLOCAL: + LOCATION: [[[], []], [[], []], [[6, 13, 20], []], [[], []]] + GROUP: [[1, 1], [1, 1], [1, 1], [1, 1]] + INSTANTIATION: dot_product + POOL: [[[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]], [[2, 2, 2], [2, 2, 2]]] +BN: + USE_PRECISE_STATS: False + NUM_BATCHES_PRECISE: 200 +SOLVER: + MOMENTUM: 0.9 + WEIGHT_DECAY: 1e-7 + OPTIMIZING_METHOD: sgd +MODEL: + NUM_CLASSES: 80 + ARCH: slowfast + MODEL_NAME: SlowFast + LOSS_FUNC: bce + DROPOUT_RATE: 0.5 + HEAD_ACT: sigmoid +TEST: + ENABLE: False + DATASET: ava + BATCH_SIZE: 8 +DATA_LOADER: + NUM_WORKERS: 2 + PIN_MEMORY: True + +NUM_GPUS: 1 +NUM_SHARDS: 1 +RNG_SEED: 0 +OUTPUT_DIR: . +#TENSORBOARD: +# MODEL_VIS: +# TOPK: 2 +DEMO: + ENABLE: True + LABEL_FILE_PATH: "/home/SlowFast-master/demo/AVA/ava.json" + INPUT_VIDEO: "/home/SlowFast-master/Vinput/clip4_003.mp4" + OUTPUT_FILE: "/home/SlowFast-master/Voutput/clip4_003.mp4" + + DETECTRON2_CFG: "COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml" + DETECTRON2_WEIGHTS: detectron2://COCO-Detection/faster_rcnn_R_50_FPN_3x/137849458/model_final_280758.pkl diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/demo/AVA/ava.json" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/demo/AVA/ava.json" new file mode 100644 index 0000000..40126da --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/demo/AVA/ava.json" @@ -0,0 +1 @@ +{"bend/bow (at the waist)": 0, "crawl": 1, "crouch/kneel": 2, "dance": 3, "fall down": 4, "get up": 5, "jump/leap": 6, "lie/sleep": 7, "martial art": 8, "run/jog": 9, "sit": 10, "stand": 11, "swim": 12, "walk": 13, "answer phone": 14, "brush teeth": 15, "carry/hold (an object)": 16, "catch (an object)": 17, "chop": 18, "climb (e.g., a mountain)": 19, "clink glass": 20, "close (e.g., a door, a box)": 21, "cook": 22, "cut": 23, "dig": 24, "dress/put on clothing": 25, "drink": 26, "drive (e.g., a car, a truck)": 27, "eat": 28, "enter": 29, "exit": 30, "extract": 31, "fishing": 32, "hit (an object)": 33, "kick (an object)": 34, "lift/pick up": 35, "listen (e.g., to music)": 36, "open (e.g., a window, a car door)": 37, "paint": 38, "play board game": 39, "play musical instrument": 40, "play with pets": 41, "point to (an object)": 42, "press": 43, "pull (an object)": 44, "push (an object)": 45, "put down": 46, "read": 47, "ride (e.g., a bike, a car, a horse)": 48, "row boat": 49, "sail boat": 50, "shoot": 51, "shovel": 52, "smoke": 53, "stir": 54, "take a photo": 55, "text on/look at a cellphone": 56, "throw": 57, "touch (an object)": 58, "turn (e.g., a screwdriver)": 59, "watch (e.g., TV)": 60, "work on a computer": 61, "write": 62, "fight/hit (a person)": 63, "give/serve (an object) to (a person)": 64, "grab (a person)": 65, "hand clap": 66, "hand shake": 67, "hand wave": 68, "hug (a person)": 69, "kick (a person)": 70, "kiss (a person)": 71, "lift (a person)": 72, "listen to (a person)": 73, "play with kids": 74, "push (another person)": 75, "sing to (e.g., self, a person, a group)": 76, "take (an object) from (a person)": 77, "talk to (e.g., self, a person, a group)": 78, "watch (a person)": 79} diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/__init__.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/__init__.py" new file mode 100644 index 0000000..b65a8f9 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/__init__.py" @@ -0,0 +1,6 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +from slowfast.utils.env import setup_environment + +setup_environment() diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/config/__init__.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/config/__init__.py" new file mode 100644 index 0000000..8dbe96a --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/config/__init__.py" @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/config/custom_config.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/config/custom_config.py" new file mode 100644 index 0000000..8131da2 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/config/custom_config.py" @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Add custom configs and default values""" + + +def add_custom_config(_C): + # Add your own customized configs. + pass diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/config/defaults.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/config/defaults.py" new file mode 100644 index 0000000..034d552 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/config/defaults.py" @@ -0,0 +1,960 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Configs.""" +from fvcore.common.config import CfgNode + +from . import custom_config + +# ----------------------------------------------------------------------------- +# Config definition +# ----------------------------------------------------------------------------- +_C = CfgNode() + +# ---------------------------------------------------------------------------- # +# Batch norm options +# ---------------------------------------------------------------------------- # +_C.BN = CfgNode() + +# Precise BN stats. +_C.BN.USE_PRECISE_STATS = False + +# Number of samples use to compute precise bn. +_C.BN.NUM_BATCHES_PRECISE = 200 + +# Weight decay value that applies on BN. +_C.BN.WEIGHT_DECAY = 0.0 + +# Norm type, options include `batchnorm`, `sub_batchnorm`, `sync_batchnorm` +_C.BN.NORM_TYPE = "batchnorm" + +# Parameter for SubBatchNorm, where it splits the batch dimension into +# NUM_SPLITS splits, and run BN on each of them separately independently. +_C.BN.NUM_SPLITS = 1 + +# Parameter for NaiveSyncBatchNorm3d, where the stats across `NUM_SYNC_DEVICES` +# devices will be synchronized. +_C.BN.NUM_SYNC_DEVICES = 1 + + +# ---------------------------------------------------------------------------- # +# Training options. +# ---------------------------------------------------------------------------- # +_C.TRAIN = CfgNode() + +# If True Train the model, else skip training. +_C.TRAIN.ENABLE = True + +# Dataset. +_C.TRAIN.DATASET = "kinetics" + +# Total mini-batch size. +_C.TRAIN.BATCH_SIZE = 64 + +# Evaluate model on test data every eval period epochs. +_C.TRAIN.EVAL_PERIOD = 10 + +# Save model checkpoint every checkpoint period epochs. +_C.TRAIN.CHECKPOINT_PERIOD = 10 + +# Resume training from the latest checkpoint in the output directory. +_C.TRAIN.AUTO_RESUME = True + +# Path to the checkpoint to load the initial weight. +_C.TRAIN.CHECKPOINT_FILE_PATH = "" + +# Checkpoint types include `caffe2` or `pytorch`. +_C.TRAIN.CHECKPOINT_TYPE = "pytorch" + +# If True, perform inflation when loading checkpoint. +_C.TRAIN.CHECKPOINT_INFLATE = False + +# If True, reset epochs when loading checkpoint. +_C.TRAIN.CHECKPOINT_EPOCH_RESET = False + +# If set, clear all layer names according to the pattern provided. +_C.TRAIN.CHECKPOINT_CLEAR_NAME_PATTERN = () # ("backbone.",) + + +# ---------------------------------------------------------------------------- # +# Augmentation options. +# ---------------------------------------------------------------------------- # +_C.AUG = CfgNode() + +# Whether to enable randaug. +_C.AUG.ENABLE = False + +# Number of repeated augmentations to used during training. +# If this is greater than 1, then the actual batch size is +# TRAIN.BATCH_SIZE * AUG.NUM_SAMPLE. +_C.AUG.NUM_SAMPLE = 1 + +# Not used if using randaug. +_C.AUG.COLOR_JITTER = 0.4 + +# RandAug parameters. +_C.AUG.AA_TYPE = "rand-m9-mstd0.5-inc1" + +# Interpolation method. +_C.AUG.INTERPOLATION = "bicubic" + +# Probability of random erasing. +_C.AUG.RE_PROB = 0.25 + +# Random erasing mode. +_C.AUG.RE_MODE = "pixel" + +# Random erase count. +_C.AUG.RE_COUNT = 1 + +# Do not random erase first (clean) augmentation split. +_C.AUG.RE_SPLIT = False + +# ---------------------------------------------------------------------------- # +# MipUp options. +# ---------------------------------------------------------------------------- # +_C.MIXUP = CfgNode() + +# Whether to use mixup. +_C.MIXUP.ENABLE = False + +# Mixup alpha. +_C.MIXUP.ALPHA = 0.8 + +# Cutmix alpha. +_C.MIXUP.CUTMIX_ALPHA = 1.0 + +# Probability of performing mixup or cutmix when either/both is enabled. +_C.MIXUP.PROB = 1.0 + +# Probability of switching to cutmix when both mixup and cutmix enabled. +_C.MIXUP.SWITCH_PROB = 0.5 + +# Label smoothing. +_C.MIXUP.LABEL_SMOOTH_VALUE = 0.1 + +# ---------------------------------------------------------------------------- # +# Testing options +# ---------------------------------------------------------------------------- # +_C.TEST = CfgNode() + +# If True test the model, else skip the testing. +_C.TEST.ENABLE = True + +# Dataset for testing. +_C.TEST.DATASET = "kinetics" + +# Total mini-batch size +_C.TEST.BATCH_SIZE = 8 + +# Path to the checkpoint to load the initial weight. +_C.TEST.CHECKPOINT_FILE_PATH = "" + +# Number of clips to sample from a video uniformly for aggregating the +# prediction results. +_C.TEST.NUM_ENSEMBLE_VIEWS = 10 + +# Number of crops to sample from a frame spatially for aggregating the +# prediction results. +_C.TEST.NUM_SPATIAL_CROPS = 3 + +# Checkpoint types include `caffe2` or `pytorch`. +_C.TEST.CHECKPOINT_TYPE = "pytorch" +# Path to saving prediction results file. +_C.TEST.SAVE_RESULTS_PATH = "" +# ----------------------------------------------------------------------------- +# ResNet options +# ----------------------------------------------------------------------------- +_C.RESNET = CfgNode() + +# Transformation function. +_C.RESNET.TRANS_FUNC = "bottleneck_transform" + +# Number of groups. 1 for ResNet, and larger than 1 for ResNeXt). +_C.RESNET.NUM_GROUPS = 1 + +# Width of each group (64 -> ResNet; 4 -> ResNeXt). +_C.RESNET.WIDTH_PER_GROUP = 64 + +# Apply relu in a inplace manner. +_C.RESNET.INPLACE_RELU = True + +# Apply stride to 1x1 conv. +_C.RESNET.STRIDE_1X1 = False + +# If true, initialize the gamma of the final BN of each block to zero. +_C.RESNET.ZERO_INIT_FINAL_BN = False + +# Number of weight layers. +_C.RESNET.DEPTH = 50 + +# If the current block has more than NUM_BLOCK_TEMP_KERNEL blocks, use temporal +# kernel of 1 for the rest of the blocks. +_C.RESNET.NUM_BLOCK_TEMP_KERNEL = [[3], [4], [6], [3]] + +# Size of stride on different res stages. +_C.RESNET.SPATIAL_STRIDES = [[1], [2], [2], [2]] + +# Size of dilation on different res stages. +_C.RESNET.SPATIAL_DILATIONS = [[1], [1], [1], [1]] + +# ---------------------------------------------------------------------------- # +# X3D options +# See https://arxiv.org/abs/2004.04730 for details about X3D Networks. +# ---------------------------------------------------------------------------- # +_C.X3D = CfgNode() + +# Width expansion factor. +_C.X3D.WIDTH_FACTOR = 1.0 + +# Depth expansion factor. +_C.X3D.DEPTH_FACTOR = 1.0 + +# Bottleneck expansion factor for the 3x3x3 conv. +_C.X3D.BOTTLENECK_FACTOR = 1.0 # + +# Dimensions of the last linear layer before classificaiton. +_C.X3D.DIM_C5 = 2048 + +# Dimensions of the first 3x3 conv layer. +_C.X3D.DIM_C1 = 12 + +# Whether to scale the width of Res2, default is false. +_C.X3D.SCALE_RES2 = False + +# Whether to use a BatchNorm (BN) layer before the classifier, default is false. +_C.X3D.BN_LIN5 = False + +# Whether to use channelwise (=depthwise) convolution in the center (3x3x3) +# convolution operation of the residual blocks. +_C.X3D.CHANNELWISE_3x3x3 = True + +# ----------------------------------------------------------------------------- +# Nonlocal options +# ----------------------------------------------------------------------------- +_C.NONLOCAL = CfgNode() + +# Index of each stage and block to add nonlocal layers. +_C.NONLOCAL.LOCATION = [[[]], [[]], [[]], [[]]] + +# Number of group for nonlocal for each stage. +_C.NONLOCAL.GROUP = [[1], [1], [1], [1]] + +# Instatiation to use for non-local layer. +_C.NONLOCAL.INSTANTIATION = "dot_product" + + +# Size of pooling layers used in Non-Local. +_C.NONLOCAL.POOL = [ + # Res2 + [[1, 2, 2], [1, 2, 2]], + # Res3 + [[1, 2, 2], [1, 2, 2]], + # Res4 + [[1, 2, 2], [1, 2, 2]], + # Res5 + [[1, 2, 2], [1, 2, 2]], +] + +# ----------------------------------------------------------------------------- +# Model options +# ----------------------------------------------------------------------------- +_C.MODEL = CfgNode() + +# Model architecture. +_C.MODEL.ARCH = "slowfast" + +# Model name +_C.MODEL.MODEL_NAME = "SlowFast" + +# The number of classes to predict for the model. +_C.MODEL.NUM_CLASSES = 400 + +# Loss function. +_C.MODEL.LOSS_FUNC = "cross_entropy" + +# Model architectures that has one single pathway. +_C.MODEL.SINGLE_PATHWAY_ARCH = ["2d", "c2d", "i3d", "slow", "x3d", "mvit"] + +# Model architectures that has multiple pathways. +_C.MODEL.MULTI_PATHWAY_ARCH = ["slowfast"] + +# Dropout rate before final projection in the backbone. +_C.MODEL.DROPOUT_RATE = 0.5 + +# Randomly drop rate for Res-blocks, linearly increase from res2 to res5 +_C.MODEL.DROPCONNECT_RATE = 0.0 + +# The std to initialize the fc layer(s). +_C.MODEL.FC_INIT_STD = 0.01 + +# Activation layer for the output head. +_C.MODEL.HEAD_ACT = "softmax" + + +# ----------------------------------------------------------------------------- +# MViT options +# ----------------------------------------------------------------------------- +_C.MVIT = CfgNode() + +# Options include `conv`, `max`. +_C.MVIT.MODE = "conv" + +# If True, perform pool before projection in attention. +_C.MVIT.POOL_FIRST = False + +# If True, use cls embed in the network, otherwise don't use cls_embed in transformer. +_C.MVIT.CLS_EMBED_ON = True + +# Kernel size for patchtification. +_C.MVIT.PATCH_KERNEL = [3, 7, 7] + +# Stride size for patchtification. +_C.MVIT.PATCH_STRIDE = [2, 4, 4] + +# Padding size for patchtification. +_C.MVIT.PATCH_PADDING = [2, 4, 4] + +# If True, use 2d patch, otherwise use 3d patch. +_C.MVIT.PATCH_2D = False + +# Base embedding dimension for the transformer. +_C.MVIT.EMBED_DIM = 96 + +# Base num of heads for the transformer. +_C.MVIT.NUM_HEADS = 1 + +# Dimension reduction ratio for the MLP layers. +_C.MVIT.MLP_RATIO = 4.0 + +# If use, use bias term in attention fc layers. +_C.MVIT.QKV_BIAS = True + +# Drop path rate for the tranfomer. +_C.MVIT.DROPPATH_RATE = 0.1 + +# Depth of the transformer. +_C.MVIT.DEPTH = 16 + +# Normalization layer for the transformer. Only layernorm is supported now. +_C.MVIT.NORM = "layernorm" + +# Dimension multiplication at layer i. If 2.0 is used, then the next block will increase +# the dimension by 2 times. Format: [depth_i: mul_dim_ratio] +_C.MVIT.DIM_MUL = [] + +# Head number multiplication at layer i. If 2.0 is used, then the next block will +# increase the number of heads by 2 times. Format: [depth_i: head_mul_ratio] +_C.MVIT.HEAD_MUL = [] + +# Stride size for the Pool KV at layer i. +# Format: [[i, stride_t_i, stride_h_i, stride_w_i], ...,] +_C.MVIT.POOL_KV_STRIDE = [] + +# Stride size for the Pool Q at layer i. +# Format: [[i, stride_t_i, stride_h_i, stride_w_i], ...,] +_C.MVIT.POOL_Q_STRIDE = [] + +# If not None, overwrite the KV_KERNEL and Q_KERNEL size with POOL_KVQ_CONV_SIZ. +# Otherwise the kernel_size is [s + 1 if s > 1 else s for s in stride_size]. +_C.MVIT.POOL_KVQ_KERNEL = None + +# If True, perform no decay on positional embedding and cls embedding. +_C.MVIT.ZERO_DECAY_POS_CLS = True + +# If True, use norm after stem. +_C.MVIT.NORM_STEM = False + +# If True, perform separate positional embedding. +_C.MVIT.SEP_POS_EMBED = False + +# Dropout rate for the MViT backbone. +_C.MVIT.DROPOUT_RATE = 0.0 + + +# ----------------------------------------------------------------------------- +# SlowFast options +# ----------------------------------------------------------------------------- +_C.SLOWFAST = CfgNode() + +# Corresponds to the inverse of the channel reduction ratio, $\beta$ between +# the Slow and Fast pathways. +_C.SLOWFAST.BETA_INV = 8 + +# Corresponds to the frame rate reduction ratio, $\alpha$ between the Slow and +# Fast pathways. +_C.SLOWFAST.ALPHA = 8 + +# Ratio of channel dimensions between the Slow and Fast pathways. +_C.SLOWFAST.FUSION_CONV_CHANNEL_RATIO = 2 + +# Kernel dimension used for fusing information from Fast pathway to Slow +# pathway. +_C.SLOWFAST.FUSION_KERNEL_SZ = 5 + + +# ----------------------------------------------------------------------------- +# Data options +# ----------------------------------------------------------------------------- +_C.DATA = CfgNode() + +# The path to the data directory. +_C.DATA.PATH_TO_DATA_DIR = "" + +# The separator used between path and label. +_C.DATA.PATH_LABEL_SEPARATOR = " " + +# Video path prefix if any. +_C.DATA.PATH_PREFIX = "" + +# The number of frames of the input clip. +_C.DATA.NUM_FRAMES = 8 + +# The video sampling rate of the input clip. +_C.DATA.SAMPLING_RATE = 8 + +# Eigenvalues for PCA jittering. Note PCA is RGB based. +_C.DATA.TRAIN_PCA_EIGVAL = [0.225, 0.224, 0.229] + +# Eigenvectors for PCA jittering. +_C.DATA.TRAIN_PCA_EIGVEC = [ + [-0.5675, 0.7192, 0.4009], + [-0.5808, -0.0045, -0.8140], + [-0.5836, -0.6948, 0.4203], +] + +# If a imdb have been dumpped to a local file with the following format: +# `{"im_path": im_path, "class": cont_id}` +# then we can skip the construction of imdb and load it from the local file. +_C.DATA.PATH_TO_PRELOAD_IMDB = "" + +# The mean value of the video raw pixels across the R G B channels. +_C.DATA.MEAN = [0.45, 0.45, 0.45] +# List of input frame channel dimensions. + +_C.DATA.INPUT_CHANNEL_NUM = [3, 3] + +# The std value of the video raw pixels across the R G B channels. +_C.DATA.STD = [0.225, 0.225, 0.225] + +# The spatial augmentation jitter scales for training. +_C.DATA.TRAIN_JITTER_SCALES = [256, 320] + +# The relative scale range of Inception-style area based random resizing augmentation. +# If this is provided, DATA.TRAIN_JITTER_SCALES above is ignored. +_C.DATA.TRAIN_JITTER_SCALES_RELATIVE = [] + +# The relative aspect ratio range of Inception-style area based random resizing +# augmentation. +_C.DATA.TRAIN_JITTER_ASPECT_RELATIVE = [] + +# If True, perform stride length uniform temporal sampling. +_C.DATA.USE_OFFSET_SAMPLING = False + +# Whether to apply motion shift for augmentation. +_C.DATA.TRAIN_JITTER_MOTION_SHIFT = False + +# The spatial crop size for training. +_C.DATA.TRAIN_CROP_SIZE = 224 + +# The spatial crop size for testing. +_C.DATA.TEST_CROP_SIZE = 256 + +# Input videos may has different fps, convert it to the target video fps before +# frame sampling. +_C.DATA.TARGET_FPS = 30 + +# Decoding backend, options include `pyav` or `torchvision` +_C.DATA.DECODING_BACKEND = "pyav" + +# if True, sample uniformly in [1 / max_scale, 1 / min_scale] and take a +# reciprocal to get the scale. If False, take a uniform sample from +# [min_scale, max_scale]. +_C.DATA.INV_UNIFORM_SAMPLE = False + +# If True, perform random horizontal flip on the video frames during training. +_C.DATA.RANDOM_FLIP = True + +# If True, calculdate the map as metric. +_C.DATA.MULTI_LABEL = False + +# Method to perform the ensemble, options include "sum" and "max". +_C.DATA.ENSEMBLE_METHOD = "sum" + +# If True, revert the default input channel (RBG <-> BGR). +_C.DATA.REVERSE_INPUT_CHANNEL = False + + +# ---------------------------------------------------------------------------- # +# Optimizer options +# ---------------------------------------------------------------------------- # +_C.SOLVER = CfgNode() + +# Base learning rate. +_C.SOLVER.BASE_LR = 0.1 + +# Learning rate policy (see utils/lr_policy.py for options and examples). +_C.SOLVER.LR_POLICY = "cosine" + +# Final learning rates for 'cosine' policy. +_C.SOLVER.COSINE_END_LR = 0.0 + +# Exponential decay factor. +_C.SOLVER.GAMMA = 0.1 + +# Step size for 'exp' and 'cos' policies (in epochs). +_C.SOLVER.STEP_SIZE = 1 + +# Steps for 'steps_' policies (in epochs). +_C.SOLVER.STEPS = [] + +# Learning rates for 'steps_' policies. +_C.SOLVER.LRS = [] + +# Maximal number of epochs. +_C.SOLVER.MAX_EPOCH = 300 + +# Momentum. +_C.SOLVER.MOMENTUM = 0.9 + +# Momentum dampening. +_C.SOLVER.DAMPENING = 0.0 + +# Nesterov momentum. +_C.SOLVER.NESTEROV = True + +# L2 regularization. +_C.SOLVER.WEIGHT_DECAY = 1e-4 + +# Start the warm up from SOLVER.BASE_LR * SOLVER.WARMUP_FACTOR. +_C.SOLVER.WARMUP_FACTOR = 0.1 + +# Gradually warm up the SOLVER.BASE_LR over this number of epochs. +_C.SOLVER.WARMUP_EPOCHS = 0.0 + +# The start learning rate of the warm up. +_C.SOLVER.WARMUP_START_LR = 0.01 + +# Optimization method. +_C.SOLVER.OPTIMIZING_METHOD = "sgd" + +# Base learning rate is linearly scaled with NUM_SHARDS. +_C.SOLVER.BASE_LR_SCALE_NUM_SHARDS = False + +# If True, start from the peak cosine learning rate after warm up. +_C.SOLVER.COSINE_AFTER_WARMUP = False + +# If True, perform no weight decay on parameter with one dimension (bias term, etc). +_C.SOLVER.ZERO_WD_1D_PARAM = False + +# ---------------------------------------------------------------------------- # +# Misc options +# ---------------------------------------------------------------------------- # + +# Number of GPUs to use (applies to both training and testing). +_C.NUM_GPUS = 1 + +# Number of machine to use for the job. +_C.NUM_SHARDS = 1 + +# The index of the current machine. +_C.SHARD_ID = 0 + +# Output basedir. +_C.OUTPUT_DIR = "./tmp" + +# Note that non-determinism may still be present due to non-deterministic +# operator implementations in GPU operator libraries. +_C.RNG_SEED = 1 + +# Log period in iters. +_C.LOG_PERIOD = 10 + +# If True, log the model info. +_C.LOG_MODEL_INFO = True + +# Distributed backend. +_C.DIST_BACKEND = "nccl" + +# ---------------------------------------------------------------------------- # +# Benchmark options +# ---------------------------------------------------------------------------- # +_C.BENCHMARK = CfgNode() + +# Number of epochs for data loading benchmark. +_C.BENCHMARK.NUM_EPOCHS = 5 + +# Log period in iters for data loading benchmark. +_C.BENCHMARK.LOG_PERIOD = 100 + +# If True, shuffle dataloader for epoch during benchmark. +_C.BENCHMARK.SHUFFLE = True + + +# ---------------------------------------------------------------------------- # +# Common train/test data loader options +# ---------------------------------------------------------------------------- # +_C.DATA_LOADER = CfgNode() + +# Number of data loader workers per training process. +_C.DATA_LOADER.NUM_WORKERS = 8 + +# Load data to pinned host memory. +_C.DATA_LOADER.PIN_MEMORY = True + +# Enable multi thread decoding. +_C.DATA_LOADER.ENABLE_MULTI_THREAD_DECODE = False + + +# ---------------------------------------------------------------------------- # +# Detection options. +# ---------------------------------------------------------------------------- # +_C.DETECTION = CfgNode() + +# Whether enable video detection. +_C.DETECTION.ENABLE = False + +# Aligned version of RoI. More details can be found at slowfast/models/head_helper.py +_C.DETECTION.ALIGNED = True + +# Spatial scale factor. +_C.DETECTION.SPATIAL_SCALE_FACTOR = 16 + +# RoI tranformation resolution. +_C.DETECTION.ROI_XFORM_RESOLUTION = 7 + + +# ----------------------------------------------------------------------------- +# AVA Dataset options +# ----------------------------------------------------------------------------- +_C.AVA = CfgNode() + +# Directory path of frames. +_C.AVA.FRAME_DIR = "/mnt/fair-flash3-east/ava_trainval_frames.img/" + +# Directory path for files of frame lists. +_C.AVA.FRAME_LIST_DIR = ( + "/mnt/vol/gfsai-flash3-east/ai-group/users/haoqifan/ava/frame_list/" +) + +# Directory path for annotation files. +_C.AVA.ANNOTATION_DIR = ( + "/mnt/vol/gfsai-flash3-east/ai-group/users/haoqifan/ava/frame_list/" +) + +# Filenames of training samples list files. +_C.AVA.TRAIN_LISTS = ["train.csv"] + +# Filenames of test samples list files. +_C.AVA.TEST_LISTS = ["val.csv"] + +# Filenames of box list files for training. Note that we assume files which +# contains predicted boxes will have a suffix "predicted_boxes" in the +# filename. +_C.AVA.TRAIN_GT_BOX_LISTS = ["ava_train_v2.2.csv"] +_C.AVA.TRAIN_PREDICT_BOX_LISTS = [] + +# Filenames of box list files for test. +_C.AVA.TEST_PREDICT_BOX_LISTS = ["ava_val_predicted_boxes.csv"] + +# This option controls the score threshold for the predicted boxes to use. +_C.AVA.DETECTION_SCORE_THRESH = 0.9 + +# If use BGR as the format of input frames. +_C.AVA.BGR = False + +# Training augmentation parameters +# Whether to use color augmentation method. +_C.AVA.TRAIN_USE_COLOR_AUGMENTATION = False + +# Whether to only use PCA jitter augmentation when using color augmentation +# method (otherwise combine with color jitter method). +_C.AVA.TRAIN_PCA_JITTER_ONLY = True + +# Whether to do horizontal flipping during test. +_C.AVA.TEST_FORCE_FLIP = False + +# Whether to use full test set for validation split. +_C.AVA.FULL_TEST_ON_VAL = False + +# The name of the file to the ava label map. +_C.AVA.LABEL_MAP_FILE = "ava_action_list_v2.2_for_activitynet_2019.pbtxt" + +# The name of the file to the ava exclusion. +_C.AVA.EXCLUSION_FILE = "ava_val_excluded_timestamps_v2.2.csv" + +# The name of the file to the ava groundtruth. +_C.AVA.GROUNDTRUTH_FILE = "ava_val_v2.2.csv" + +# Backend to process image, includes `pytorch` and `cv2`. +_C.AVA.IMG_PROC_BACKEND = "cv2" + +# ---------------------------------------------------------------------------- # +# Multigrid training options +# See https://arxiv.org/abs/1912.00998 for details about multigrid training. +# ---------------------------------------------------------------------------- # +_C.MULTIGRID = CfgNode() + +# Multigrid training allows us to train for more epochs with fewer iterations. +# This hyperparameter specifies how many times more epochs to train. +# The default setting in paper trains for 1.5x more epochs than baseline. +_C.MULTIGRID.EPOCH_FACTOR = 1.5 + +# Enable short cycles. +_C.MULTIGRID.SHORT_CYCLE = False +# Short cycle additional spatial dimensions relative to the default crop size. +_C.MULTIGRID.SHORT_CYCLE_FACTORS = [0.5, 0.5 ** 0.5] + +_C.MULTIGRID.LONG_CYCLE = False +# (Temporal, Spatial) dimensions relative to the default shape. +_C.MULTIGRID.LONG_CYCLE_FACTORS = [ + (0.25, 0.5 ** 0.5), + (0.5, 0.5 ** 0.5), + (0.5, 1), + (1, 1), +] + +# While a standard BN computes stats across all examples in a GPU, +# for multigrid training we fix the number of clips to compute BN stats on. +# See https://arxiv.org/abs/1912.00998 for details. +_C.MULTIGRID.BN_BASE_SIZE = 8 + +# Multigrid training epochs are not proportional to actual training time or +# computations, so _C.TRAIN.EVAL_PERIOD leads to too frequent or rare +# evaluation. We use a multigrid-specific rule to determine when to evaluate: +# This hyperparameter defines how many times to evaluate a model per long +# cycle shape. +_C.MULTIGRID.EVAL_FREQ = 3 + +# No need to specify; Set automatically and used as global variables. +_C.MULTIGRID.LONG_CYCLE_SAMPLING_RATE = 0 +_C.MULTIGRID.DEFAULT_B = 0 +_C.MULTIGRID.DEFAULT_T = 0 +_C.MULTIGRID.DEFAULT_S = 0 + +# ----------------------------------------------------------------------------- +# Tensorboard Visualization Options +# ----------------------------------------------------------------------------- +_C.TENSORBOARD = CfgNode() + +# Log to summary writer, this will automatically. +# log loss, lr and metrics during train/eval. +_C.TENSORBOARD.ENABLE = False +# Provide path to prediction results for visualization. +# This is a pickle file of [prediction_tensor, label_tensor] +_C.TENSORBOARD.PREDICTIONS_PATH = "" +# Path to directory for tensorboard logs. +# Default to to cfg.OUTPUT_DIR/runs-{cfg.TRAIN.DATASET}. +_C.TENSORBOARD.LOG_DIR = "" +# Path to a json file providing class_name - id mapping +# in the format {"class_name1": id1, "class_name2": id2, ...}. +# This file must be provided to enable plotting confusion matrix +# by a subset or parent categories. +_C.TENSORBOARD.CLASS_NAMES_PATH = "" + +# Path to a json file for categories -> classes mapping +# in the format {"parent_class": ["child_class1", "child_class2",...], ...}. +_C.TENSORBOARD.CATEGORIES_PATH = "" + +# Config for confusion matrices visualization. +_C.TENSORBOARD.CONFUSION_MATRIX = CfgNode() +# Visualize confusion matrix. +_C.TENSORBOARD.CONFUSION_MATRIX.ENABLE = False +# Figure size of the confusion matrices plotted. +_C.TENSORBOARD.CONFUSION_MATRIX.FIGSIZE = [8, 8] +# Path to a subset of categories to visualize. +# File contains class names separated by newline characters. +_C.TENSORBOARD.CONFUSION_MATRIX.SUBSET_PATH = "" + +# Config for histogram visualization. +_C.TENSORBOARD.HISTOGRAM = CfgNode() +# Visualize histograms. +_C.TENSORBOARD.HISTOGRAM.ENABLE = False +# Path to a subset of classes to plot histograms. +# Class names must be separated by newline characters. +_C.TENSORBOARD.HISTOGRAM.SUBSET_PATH = "" +# Visualize top-k most predicted classes on histograms for each +# chosen true label. +_C.TENSORBOARD.HISTOGRAM.TOPK = 10 +# Figure size of the histograms plotted. +_C.TENSORBOARD.HISTOGRAM.FIGSIZE = [8, 8] + +# Config for layers' weights and activations visualization. +# _C.TENSORBOARD.ENABLE must be True. +_C.TENSORBOARD.MODEL_VIS = CfgNode() + +# If False, skip model visualization. +_C.TENSORBOARD.MODEL_VIS.ENABLE = False + +# If False, skip visualizing model weights. +_C.TENSORBOARD.MODEL_VIS.MODEL_WEIGHTS = False + +# If False, skip visualizing model activations. +_C.TENSORBOARD.MODEL_VIS.ACTIVATIONS = False + +# If False, skip visualizing input videos. +_C.TENSORBOARD.MODEL_VIS.INPUT_VIDEO = False + + +# List of strings containing data about layer names and their indexing to +# visualize weights and activations for. The indexing is meant for +# choosing a subset of activations outputed by a layer for visualization. +# If indexing is not specified, visualize all activations outputed by the layer. +# For each string, layer name and indexing is separated by whitespaces. +# e.g.: [layer1 1,2;1,2, layer2, layer3 150,151;3,4]; this means for each array `arr` +# along the batch dimension in `layer1`, we take arr[[1, 2], [1, 2]] +_C.TENSORBOARD.MODEL_VIS.LAYER_LIST = [] +# Top-k predictions to plot on videos +_C.TENSORBOARD.MODEL_VIS.TOPK_PREDS = 1 +# Colormap to for text boxes and bounding boxes colors +_C.TENSORBOARD.MODEL_VIS.COLORMAP = "Pastel2" +# Config for visualization video inputs with Grad-CAM. +# _C.TENSORBOARD.ENABLE must be True. +_C.TENSORBOARD.MODEL_VIS.GRAD_CAM = CfgNode() +# Whether to run visualization using Grad-CAM technique. +_C.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE = True +# CNN layers to use for Grad-CAM. The number of layers must be equal to +# number of pathway(s). +_C.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST = [] +# If True, visualize Grad-CAM using true labels for each instances. +# If False, use the highest predicted class. +_C.TENSORBOARD.MODEL_VIS.GRAD_CAM.USE_TRUE_LABEL = False +# Colormap to for text boxes and bounding boxes colors +_C.TENSORBOARD.MODEL_VIS.GRAD_CAM.COLORMAP = "viridis" + +# Config for visualization for wrong prediction visualization. +# _C.TENSORBOARD.ENABLE must be True. +_C.TENSORBOARD.WRONG_PRED_VIS = CfgNode() +_C.TENSORBOARD.WRONG_PRED_VIS.ENABLE = False +# Folder tag to origanize model eval videos under. +_C.TENSORBOARD.WRONG_PRED_VIS.TAG = "Incorrectly classified videos." +# Subset of labels to visualize. Only wrong predictions with true labels +# within this subset is visualized. +_C.TENSORBOARD.WRONG_PRED_VIS.SUBSET_PATH = "" + + +# ---------------------------------------------------------------------------- # +# Demo options +# ---------------------------------------------------------------------------- # +_C.DEMO = CfgNode() + +# Run model in DEMO mode. +_C.DEMO.ENABLE = False + +# Path to a json file providing class_name - id mapping +# in the format {"class_name1": id1, "class_name2": id2, ...}. +_C.DEMO.LABEL_FILE_PATH = "" + +# Specify a camera device as input. This will be prioritized +# over input video if set. +# If -1, use input video instead. +_C.DEMO.WEBCAM = -1 + +# Path to input video for demo. +_C.DEMO.INPUT_VIDEO = "" +# Custom width for reading input video data. +_C.DEMO.DISPLAY_WIDTH = 0 +# Custom height for reading input video data. +_C.DEMO.DISPLAY_HEIGHT = 0 +# Path to Detectron2 object detection model configuration, +# only used for detection tasks. +_C.DEMO.DETECTRON2_CFG = "COCO-Detection/faster_rcnn_R_50_FPN_3x.yaml" +# Path to Detectron2 object detection model pre-trained weights. +_C.DEMO.DETECTRON2_WEIGHTS = "detectron2://COCO-Detection/faster_rcnn_R_50_FPN_3x/137849458/model_final_280758.pkl" +# Threshold for choosing predicted bounding boxes by Detectron2. +_C.DEMO.DETECTRON2_THRESH = 0.9 +# Number of overlapping frames between 2 consecutive clips. +# Increase this number for more frequent action predictions. +# The number of overlapping frames cannot be larger than +# half of the sequence length `cfg.DATA.NUM_FRAMES * cfg.DATA.SAMPLING_RATE` +_C.DEMO.BUFFER_SIZE = 0 +# If specified, the visualized outputs will be written this a video file of +# this path. Otherwise, the visualized outputs will be displayed in a window. +_C.DEMO.OUTPUT_FILE = "" +# Frames per second rate for writing to output video file. +# If not set (-1), use fps rate from input file. +_C.DEMO.OUTPUT_FPS = -1 +# Input format from demo video reader ("RGB" or "BGR"). +_C.DEMO.INPUT_FORMAT = "BGR" +# Draw visualization frames in [keyframe_idx - CLIP_VIS_SIZE, keyframe_idx + CLIP_VIS_SIZE] inclusively. +_C.DEMO.CLIP_VIS_SIZE = 10 +# Number of processes to run video visualizer. +_C.DEMO.NUM_VIS_INSTANCES = 2 + +# Path to pre-computed predicted boxes +_C.DEMO.PREDS_BOXES = "" +# Whether to run in with multi-threaded video reader. +_C.DEMO.THREAD_ENABLE = False +# Take one clip for every `DEMO.NUM_CLIPS_SKIP` + 1 for prediction and visualization. +# This is used for fast demo speed by reducing the prediction/visualiztion frequency. +# If -1, take the most recent read clip for visualization. This mode is only supported +# if `DEMO.THREAD_ENABLE` is set to True. +_C.DEMO.NUM_CLIPS_SKIP = 0 +# Path to ground-truth boxes and labels (optional) +_C.DEMO.GT_BOXES = "" +# The starting second of the video w.r.t bounding boxes file. +_C.DEMO.STARTING_SECOND = 900 +# Frames per second of the input video/folder of images. +_C.DEMO.FPS = 30 +# Visualize with top-k predictions or predictions above certain threshold(s). +# Option: {"thres", "top-k"} +_C.DEMO.VIS_MODE = "thres" +# Threshold for common class names. +_C.DEMO.COMMON_CLASS_THRES = 0.7 +# Theshold for uncommon class names. This will not be +# used if `_C.DEMO.COMMON_CLASS_NAMES` is empty. +_C.DEMO.UNCOMMON_CLASS_THRES = 0.3 +# This is chosen based on distribution of examples in +# each classes in AVA dataset. +_C.DEMO.COMMON_CLASS_NAMES = [ + "watch (a person)", + "talk to (e.g., self, a person, a group)", + "listen to (a person)", + "touch (an object)", + "carry/hold (an object)", + "walk", + "sit", + "lie/sleep", + "bend/bow (at the waist)", +] +# Slow-motion rate for the visualization. The visualized portions of the +# video will be played `_C.DEMO.SLOWMO` times slower than usual speed. +_C.DEMO.SLOWMO = 1 + +# Add custom config with default values. +custom_config.add_custom_config(_C) + + +def assert_and_infer_cfg(cfg): + # BN assertions. + if cfg.BN.USE_PRECISE_STATS: + assert cfg.BN.NUM_BATCHES_PRECISE >= 0 + # TRAIN assertions. + assert cfg.TRAIN.CHECKPOINT_TYPE in ["pytorch", "caffe2"] + assert cfg.TRAIN.BATCH_SIZE % cfg.NUM_GPUS == 0 + + # TEST assertions. + assert cfg.TEST.CHECKPOINT_TYPE in ["pytorch", "caffe2"] + assert cfg.TEST.BATCH_SIZE % cfg.NUM_GPUS == 0 + + # RESNET assertions. + assert cfg.RESNET.NUM_GROUPS > 0 + assert cfg.RESNET.WIDTH_PER_GROUP > 0 + assert cfg.RESNET.WIDTH_PER_GROUP % cfg.RESNET.NUM_GROUPS == 0 + + # Execute LR scaling by num_shards. + if cfg.SOLVER.BASE_LR_SCALE_NUM_SHARDS: + cfg.SOLVER.BASE_LR *= cfg.NUM_SHARDS + cfg.SOLVER.WARMUP_START_LR *= cfg.NUM_SHARDS + cfg.SOLVER.COSINE_END_LR *= cfg.NUM_SHARDS + + # General assertions. + assert cfg.SHARD_ID < cfg.NUM_SHARDS + return cfg + + +def get_cfg(): + """ + Get a copy of the default config. + """ + return _C.clone() diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/DATASET.md" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/DATASET.md" new file mode 100644 index 0000000..4190400 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/DATASET.md" @@ -0,0 +1,165 @@ +# Dataset Preparation + +## Kinetics + +The Kinetics Dataset could be downloaded via the code released by ActivityNet: + +1. Download the videos via the official [scripts](https://github.com/activitynet/ActivityNet/tree/master/Crawler/Kinetics). + +2. After all the videos were downloaded, resize the video to the short edge size of 256, then prepare the csv files for training, validation, and testing set as `train.csv`, `val.csv`, `test.csv`. The format of the csv file is: + +``` +path_to_video_1 label_1 +path_to_video_2 label_2 +path_to_video_3 label_3 +... +path_to_video_N label_N +``` + +All the Kinetics models in the Model Zoo are trained and tested with the same data as [Non-local Network](https://github.com/facebookresearch/video-nonlocal-net/blob/master/DATASET.md). For dataset specific issues, please reach out to the [dataset provider](https://deepmind.com/research/open-source/kinetics). + +## AVA + +The AVA Dataset could be downloaded from the [official site](https://research.google.com/ava/download.html#ava_actions_download) + +We followed the same [downloading and preprocessing procedure](https://github.com/facebookresearch/video-long-term-feature-banks/blob/master/DATASET.md) as the [Long-Term Feature Banks for Detailed Video Understanding](https://arxiv.org/abs/1812.05038) do. + +You could follow these steps to download and preprocess the data: + +1. Download videos + +``` +DATA_DIR="../../data/ava/videos" + +if [[ ! -d "${DATA_DIR}" ]]; then + echo "${DATA_DIR} doesn't exist. Creating it."; + mkdir -p ${DATA_DIR} +fi + +wget https://s3.amazonaws.com/ava-dataset/annotations/ava_file_names_trainval_v2.1.txt + +for line in $(cat ava_file_names_trainval_v2.1.txt) +do + wget https://s3.amazonaws.com/ava-dataset/trainval/$line -P ${DATA_DIR} +done +``` + +2. Cut each video from its 15th to 30th minute + +``` +IN_DATA_DIR="../../data/ava/videos" +OUT_DATA_DIR="../../data/ava/videos_15min" + +if [[ ! -d "${OUT_DATA_DIR}" ]]; then + echo "${OUT_DATA_DIR} doesn't exist. Creating it."; + mkdir -p ${OUT_DATA_DIR} +fi + +for video in $(ls -A1 -U ${IN_DATA_DIR}/*) +do + out_name="${OUT_DATA_DIR}/${video##*/}" + if [ ! -f "${out_name}" ]; then + ffmpeg -ss 900 -t 901 -i "${video}" "${out_name}" + fi +done +``` + +3. Extract frames + +``` +IN_DATA_DIR="../../data/ava/videos_15min" +OUT_DATA_DIR="../../data/ava/frames" + +if [[ ! -d "${OUT_DATA_DIR}" ]]; then + echo "${OUT_DATA_DIR} doesn't exist. Creating it."; + mkdir -p ${OUT_DATA_DIR} +fi + +for video in $(ls -A1 -U ${IN_DATA_DIR}/*) +do + video_name=${video##*/} + + if [[ $video_name = *".webm" ]]; then + video_name=${video_name::-5} + else + video_name=${video_name::-4} + fi + + out_video_dir=${OUT_DATA_DIR}/${video_name}/ + mkdir -p "${out_video_dir}" + + out_name="${out_video_dir}/${video_name}_%06d.jpg" + + ffmpeg -i "${video}" -r 30 -q:v 1 "${out_name}" +done +``` + +4. Download annotations + +``` +DATA_DIR="../../data/ava/annotations" + +if [[ ! -d "${DATA_DIR}" ]]; then + echo "${DATA_DIR} doesn't exist. Creating it."; + mkdir -p ${DATA_DIR} +fi + +wget https://research.google.com/ava/download/ava_train_v2.1.csv -P ${DATA_DIR} +wget https://research.google.com/ava/download/ava_val_v2.1.csv -P ${DATA_DIR} +wget https://research.google.com/ava/download/ava_action_list_v2.1_for_activitynet_2018.pbtxt -P ${DATA_DIR} +wget https://research.google.com/ava/download/ava_train_excluded_timestamps_v2.1.csv -P ${DATA_DIR} +wget https://research.google.com/ava/download/ava_val_excluded_timestamps_v2.1.csv -P ${DATA_DIR} +``` + +5. Download "frame lists" ([train](https://dl.fbaipublicfiles.com/video-long-term-feature-banks/data/ava/frame_lists/train.csv), [val](https://dl.fbaipublicfiles.com/video-long-term-feature-banks/data/ava/frame_lists/val.csv)) and put them in +the `frame_lists` folder (see structure above). + +6. Download person boxes ([train](https://dl.fbaipublicfiles.com/video-long-term-feature-banks/data/ava/annotations/ava_train_predicted_boxes.csv), [val](https://dl.fbaipublicfiles.com/video-long-term-feature-banks/data/ava/annotations/ava_val_predicted_boxes.csv), [test](https://dl.fbaipublicfiles.com/video-long-term-feature-banks/data/ava/annotations/ava_test_predicted_boxes.csv)) and put them in the `annotations` folder (see structure above). +If you prefer to use your own person detector, please see details +in [here](https://github.com/facebookresearch/video-long-term-feature-banks/blob/master/GETTING_STARTED.md#ava-person-detector). + + +Download the ava dataset with the following structure: + +``` +ava +|_ frames +| |_ [video name 0] +| | |_ [video name 0]_000001.jpg +| | |_ [video name 0]_000002.jpg +| | |_ ... +| |_ [video name 1] +| |_ [video name 1]_000001.jpg +| |_ [video name 1]_000002.jpg +| |_ ... +|_ frame_lists +| |_ train.csv +| |_ val.csv +|_ annotations + |_ [official AVA annotation files] + |_ ava_train_predicted_boxes.csv + |_ ava_val_predicted_boxes.csv +``` + +You could also replace the `v2.1` by `v2.2` if you need the AVA v2.2 annotation. You can also download some pre-prepared annotations from [here](https://dl.fbaipublicfiles.com/pyslowfast/annotation/ava/ava_annotations.tar). + + +## Charades +1. Please download the Charades RGB frames from [dataset provider](http://ai2-website.s3.amazonaws.com/data/Charades_v1_rgb.tar). + +2. Download the *frame list* from the following links: ([train](https://dl.fbaipublicfiles.com/pyslowfast/dataset/charades/frame_lists/train.csv), [val](https://dl.fbaipublicfiles.com/pyslowfast/dataset/charades/frame_lists/val.csv)). + +Please set `DATA.PATH_TO_DATA_DIR` to point to the folder containing the frame lists, and `DATA.PATH_PREFIX` to the folder containing RGB frames. + + +## Something-Something V2 +1. Please download the dataset and annotations from [dataset provider](https://20bn.com/datasets/something-something). + +2. Download the *frame list* from the following links: ([train](https://dl.fbaipublicfiles.com/pyslowfast/dataset/ssv2/frame_lists/train.csv), [val](https://dl.fbaipublicfiles.com/pyslowfast/dataset/ssv2/frame_lists/val.csv)). + +3. Extract the frames at 30 FPS. (We used ffmpeg-4.1.3 with command +`ffmpeg -i "${video}" -r 30 -q:v 1 "${out_name}"` + in experiments.) Please put the frames in a structure consistent with the frame lists. + + +Please put all annotation json files and the frame lists in the same folder, and set `DATA.PATH_TO_DATA_DIR` to the path. Set `DATA.PATH_PREFIX` to be the path to the folder containing extracted frames. diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/__init__.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/__init__.py" new file mode 100644 index 0000000..dee6342 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/__init__.py" @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +from .ava_dataset import Ava # noqa +from .build import DATASET_REGISTRY, build_dataset # noqa +from .charades import Charades # noqa +from .imagenet import Imagenet # noqa +from .kinetics import Kinetics # noqa +from .ssv2 import Ssv2 # noqa +try: + from .ptv_datasets import Ptvcharades, Ptvkinetics, Ptvssv2 # noqa +except Exception: + print("Please update your PyTorchVideo to latest master") diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ava_dataset.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ava_dataset.py" new file mode 100644 index 0000000..4a55635 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ava_dataset.py" @@ -0,0 +1,428 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import logging +import numpy as np +import torch + +from . import ava_helper as ava_helper +from . import cv2_transform as cv2_transform +from . import transform as transform +from . import utils as utils +from .build import DATASET_REGISTRY + +logger = logging.getLogger(__name__) + + +@DATASET_REGISTRY.register() +class Ava(torch.utils.data.Dataset): + """ + AVA Dataset + """ + + def __init__(self, cfg, split): + self.cfg = cfg + self._split = split + self._sample_rate = cfg.DATA.SAMPLING_RATE + self._video_length = cfg.DATA.NUM_FRAMES + self._seq_len = self._video_length * self._sample_rate + self._num_classes = cfg.MODEL.NUM_CLASSES + # Augmentation params. + self._data_mean = cfg.DATA.MEAN + self._data_std = cfg.DATA.STD + self._use_bgr = cfg.AVA.BGR + self.random_horizontal_flip = cfg.DATA.RANDOM_FLIP + if self._split == "train": + self._crop_size = cfg.DATA.TRAIN_CROP_SIZE + self._jitter_min_scale = cfg.DATA.TRAIN_JITTER_SCALES[0] + self._jitter_max_scale = cfg.DATA.TRAIN_JITTER_SCALES[1] + self._use_color_augmentation = cfg.AVA.TRAIN_USE_COLOR_AUGMENTATION + self._pca_jitter_only = cfg.AVA.TRAIN_PCA_JITTER_ONLY + self._pca_eigval = cfg.DATA.TRAIN_PCA_EIGVAL + self._pca_eigvec = cfg.DATA.TRAIN_PCA_EIGVEC + else: + self._crop_size = cfg.DATA.TEST_CROP_SIZE + self._test_force_flip = cfg.AVA.TEST_FORCE_FLIP + + self._load_data(cfg) + + def _load_data(self, cfg): + """ + Load frame paths and annotations from files + + Args: + cfg (CfgNode): config + """ + # Loading frame paths. + ( + self._image_paths, + self._video_idx_to_name, + ) = ava_helper.load_image_lists(cfg, is_train=(self._split == "train")) + + # Loading annotations for boxes and labels. + boxes_and_labels = ava_helper.load_boxes_and_labels( + cfg, mode=self._split + ) + + assert len(boxes_and_labels) == len(self._image_paths) + + boxes_and_labels = [ + boxes_and_labels[self._video_idx_to_name[i]] + for i in range(len(self._image_paths)) + ] + + # Get indices of keyframes and corresponding boxes and labels. + ( + self._keyframe_indices, + self._keyframe_boxes_and_labels, + ) = ava_helper.get_keyframe_data(boxes_and_labels) + + # Calculate the number of used boxes. + self._num_boxes_used = ava_helper.get_num_boxes_used( + self._keyframe_indices, self._keyframe_boxes_and_labels + ) + + self.print_summary() + + def print_summary(self): + logger.info("=== AVA dataset summary ===") + logger.info("Split: {}".format(self._split)) + logger.info("Number of videos: {}".format(len(self._image_paths))) + total_frames = sum( + len(video_img_paths) for video_img_paths in self._image_paths + ) + logger.info("Number of frames: {}".format(total_frames)) + logger.info("Number of key frames: {}".format(len(self))) + logger.info("Number of boxes: {}.".format(self._num_boxes_used)) + + def __len__(self): + """ + Returns: + (int): the number of videos in the dataset. + """ + return self.num_videos + + @property + def num_videos(self): + """ + Returns: + (int): the number of videos in the dataset. + """ + return len(self._keyframe_indices) + + def _images_and_boxes_preprocessing_cv2(self, imgs, boxes): + """ + This function performs preprocessing for the input images and + corresponding boxes for one clip with opencv as backend. + + Args: + imgs (tensor): the images. + boxes (ndarray): the boxes for the current clip. + + Returns: + imgs (tensor): list of preprocessed images. + boxes (ndarray): preprocessed boxes. + """ + + height, width, _ = imgs[0].shape + + boxes[:, [0, 2]] *= width + boxes[:, [1, 3]] *= height + boxes = cv2_transform.clip_boxes_to_image(boxes, height, width) + + # `transform.py` is list of np.array. However, for AVA, we only have + # one np.array. + boxes = [boxes] + + # The image now is in HWC, BGR format. + if self._split == "train": # "train" + imgs, boxes = cv2_transform.random_short_side_scale_jitter_list( + imgs, + min_size=self._jitter_min_scale, + max_size=self._jitter_max_scale, + boxes=boxes, + ) + imgs, boxes = cv2_transform.random_crop_list( + imgs, self._crop_size, order="HWC", boxes=boxes + ) + + if self.random_horizontal_flip: + # random flip + imgs, boxes = cv2_transform.horizontal_flip_list( + 0.5, imgs, order="HWC", boxes=boxes + ) + elif self._split == "val": + # Short side to test_scale. Non-local and STRG uses 256. + imgs = [cv2_transform.scale(self._crop_size, img) for img in imgs] + boxes = [ + cv2_transform.scale_boxes( + self._crop_size, boxes[0], height, width + ) + ] + imgs, boxes = cv2_transform.spatial_shift_crop_list( + self._crop_size, imgs, 1, boxes=boxes + ) + + if self._test_force_flip: + imgs, boxes = cv2_transform.horizontal_flip_list( + 1, imgs, order="HWC", boxes=boxes + ) + elif self._split == "test": + # Short side to test_scale. Non-local and STRG uses 256. + imgs = [cv2_transform.scale(self._crop_size, img) for img in imgs] + boxes = [ + cv2_transform.scale_boxes( + self._crop_size, boxes[0], height, width + ) + ] + + if self._test_force_flip: + imgs, boxes = cv2_transform.horizontal_flip_list( + 1, imgs, order="HWC", boxes=boxes + ) + else: + raise NotImplementedError( + "Unsupported split mode {}".format(self._split) + ) + + # Convert image to CHW keeping BGR order. + imgs = [cv2_transform.HWC2CHW(img) for img in imgs] + + # Image [0, 255] -> [0, 1]. + imgs = [img / 255.0 for img in imgs] + + imgs = [ + np.ascontiguousarray( + # img.reshape((3, self._crop_size, self._crop_size)) + img.reshape((3, imgs[0].shape[1], imgs[0].shape[2])) + ).astype(np.float32) + for img in imgs + ] + + # Do color augmentation (after divided by 255.0). + if self._split == "train" and self._use_color_augmentation: + if not self._pca_jitter_only: + imgs = cv2_transform.color_jitter_list( + imgs, + img_brightness=0.4, + img_contrast=0.4, + img_saturation=0.4, + ) + + imgs = cv2_transform.lighting_list( + imgs, + alphastd=0.1, + eigval=np.array(self._pca_eigval).astype(np.float32), + eigvec=np.array(self._pca_eigvec).astype(np.float32), + ) + + # Normalize images by mean and std. + imgs = [ + cv2_transform.color_normalization( + img, + np.array(self._data_mean, dtype=np.float32), + np.array(self._data_std, dtype=np.float32), + ) + for img in imgs + ] + + # Concat list of images to single ndarray. + imgs = np.concatenate( + [np.expand_dims(img, axis=1) for img in imgs], axis=1 + ) + + if not self._use_bgr: + # Convert image format from BGR to RGB. + imgs = imgs[::-1, ...] + + imgs = np.ascontiguousarray(imgs) + imgs = torch.from_numpy(imgs) + boxes = cv2_transform.clip_boxes_to_image( + boxes[0], imgs[0].shape[1], imgs[0].shape[2] + ) + return imgs, boxes + + def _images_and_boxes_preprocessing(self, imgs, boxes): + """ + This function performs preprocessing for the input images and + corresponding boxes for one clip. + + Args: + imgs (tensor): the images. + boxes (ndarray): the boxes for the current clip. + + Returns: + imgs (tensor): list of preprocessed images. + boxes (ndarray): preprocessed boxes. + """ + # Image [0, 255] -> [0, 1]. + imgs = imgs.float() + imgs = imgs / 255.0 + + height, width = imgs.shape[2], imgs.shape[3] + # The format of boxes is [x1, y1, x2, y2]. The input boxes are in the + # range of [0, 1]. + boxes[:, [0, 2]] *= width + boxes[:, [1, 3]] *= height + boxes = transform.clip_boxes_to_image(boxes, height, width) + + if self._split == "train": + # Train split + imgs, boxes = transform.random_short_side_scale_jitter( + imgs, + min_size=self._jitter_min_scale, + max_size=self._jitter_max_scale, + boxes=boxes, + ) + imgs, boxes = transform.random_crop( + imgs, self._crop_size, boxes=boxes + ) + + # Random flip. + imgs, boxes = transform.horizontal_flip(0.5, imgs, boxes=boxes) + elif self._split == "val": + # Val split + # Resize short side to crop_size. Non-local and STRG uses 256. + imgs, boxes = transform.random_short_side_scale_jitter( + imgs, + min_size=self._crop_size, + max_size=self._crop_size, + boxes=boxes, + ) + + # Apply center crop for val split + imgs, boxes = transform.uniform_crop( + imgs, size=self._crop_size, spatial_idx=1, boxes=boxes + ) + + if self._test_force_flip: + imgs, boxes = transform.horizontal_flip(1, imgs, boxes=boxes) + elif self._split == "test": + # Test split + # Resize short side to crop_size. Non-local and STRG uses 256. + imgs, boxes = transform.random_short_side_scale_jitter( + imgs, + min_size=self._crop_size, + max_size=self._crop_size, + boxes=boxes, + ) + + if self._test_force_flip: + imgs, boxes = transform.horizontal_flip(1, imgs, boxes=boxes) + else: + raise NotImplementedError( + "{} split not supported yet!".format(self._split) + ) + + # Do color augmentation (after divided by 255.0). + if self._split == "train" and self._use_color_augmentation: + if not self._pca_jitter_only: + imgs = transform.color_jitter( + imgs, + img_brightness=0.4, + img_contrast=0.4, + img_saturation=0.4, + ) + + imgs = transform.lighting_jitter( + imgs, + alphastd=0.1, + eigval=np.array(self._pca_eigval).astype(np.float32), + eigvec=np.array(self._pca_eigvec).astype(np.float32), + ) + + # Normalize images by mean and std. + imgs = transform.color_normalization( + imgs, + np.array(self._data_mean, dtype=np.float32), + np.array(self._data_std, dtype=np.float32), + ) + + if not self._use_bgr: + # Convert image format from BGR to RGB. + # Note that Kinetics pre-training uses RGB! + imgs = imgs[:, [2, 1, 0], ...] + + boxes = transform.clip_boxes_to_image( + boxes, self._crop_size, self._crop_size + ) + + return imgs, boxes + + def __getitem__(self, idx): + """ + Generate corresponding clips, boxes, labels and metadata for given idx. + + Args: + idx (int): the video index provided by the pytorch sampler. + Returns: + frames (tensor): the frames of sampled from the video. The dimension + is `channel` x `num frames` x `height` x `width`. + label (ndarray): the label for correspond boxes for the current video. + idx (int): the video index provided by the pytorch sampler. + extra_data (dict): a dict containing extra data fields, like "boxes", + "ori_boxes" and "metadata". + """ + video_idx, sec_idx, sec, center_idx = self._keyframe_indices[idx] + # Get the frame idxs for current clip. + seq = utils.get_sequence( + center_idx, + self._seq_len // 2, + self._sample_rate, + num_frames=len(self._image_paths[video_idx]), + ) + + clip_label_list = self._keyframe_boxes_and_labels[video_idx][sec_idx] + assert len(clip_label_list) > 0 + + # Get boxes and labels for current clip. + boxes = [] + labels = [] + for box_labels in clip_label_list: + boxes.append(box_labels[0]) + labels.append(box_labels[1]) + boxes = np.array(boxes) + # Score is not used. + boxes = boxes[:, :4].copy() + ori_boxes = boxes.copy() + + # Load images of current clip. + image_paths = [self._image_paths[video_idx][frame] for frame in seq] + imgs = utils.retry_load_images( + image_paths, backend=self.cfg.AVA.IMG_PROC_BACKEND + ) + if self.cfg.AVA.IMG_PROC_BACKEND == "pytorch": + # T H W C -> T C H W. + imgs = imgs.permute(0, 3, 1, 2) + # Preprocess images and boxes. + imgs, boxes = self._images_and_boxes_preprocessing( + imgs, boxes=boxes + ) + # T C H W -> C T H W. + imgs = imgs.permute(1, 0, 2, 3) + else: + # Preprocess images and boxes + imgs, boxes = self._images_and_boxes_preprocessing_cv2( + imgs, boxes=boxes + ) + + # Construct label arrays. + label_arrs = np.zeros((len(labels), self._num_classes), dtype=np.int32) + for i, box_labels in enumerate(labels): + # AVA label index starts from 1. + for label in box_labels: + if label == -1: + continue + assert label >= 1 and label <= 80 + label_arrs[i][label - 1] = 1 + + imgs = utils.pack_pathway_output(self.cfg, imgs) + metadata = [[video_idx, sec]] * len(boxes) + + extra_data = { + "boxes": boxes, + "ori_boxes": ori_boxes, + "metadata": metadata, + } + + return imgs, label_arrs, idx, extra_data diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ava_helper.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ava_helper.py" new file mode 100644 index 0000000..ce5d8bf --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ava_helper.py" @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import logging +import os +from collections import defaultdict +from iopath.common.file_io import g_pathmgr + +logger = logging.getLogger(__name__) + +FPS = 30 +AVA_VALID_FRAMES = range(902, 1799) + + +def load_image_lists(cfg, is_train): + """ + Loading image paths from corresponding files. + + Args: + cfg (CfgNode): config. + is_train (bool): if it is training dataset or not. + + Returns: + image_paths (list[list]): a list of items. Each item (also a list) + corresponds to one video and contains the paths of images for + this video. + video_idx_to_name (list): a list which stores video names. + """ + list_filenames = [ + os.path.join(cfg.AVA.FRAME_LIST_DIR, filename) + for filename in ( + cfg.AVA.TRAIN_LISTS if is_train else cfg.AVA.TEST_LISTS + ) + ] + image_paths = defaultdict(list) + video_name_to_idx = {} + video_idx_to_name = [] + for list_filename in list_filenames: + with g_pathmgr.open(list_filename, "r") as f: + f.readline() + for line in f: + row = line.split() + # The format of each row should follow: + # original_vido_id video_id frame_id path labels. + assert len(row) == 5 + video_name = row[0] + + if video_name not in video_name_to_idx: + idx = len(video_name_to_idx) + video_name_to_idx[video_name] = idx + video_idx_to_name.append(video_name) + + data_key = video_name_to_idx[video_name] + + image_paths[data_key].append( + os.path.join(cfg.AVA.FRAME_DIR, row[3]) + ) + + image_paths = [image_paths[i] for i in range(len(image_paths))] + + logger.info( + "Finished loading image paths from: %s" % ", ".join(list_filenames) + ) + + return image_paths, video_idx_to_name + + +def load_boxes_and_labels(cfg, mode): + """ + Loading boxes and labels from csv files. + + Args: + cfg (CfgNode): config. + mode (str): 'train', 'val', or 'test' mode. + Returns: + all_boxes (dict): a dict which maps from `video_name` and + `frame_sec` to a list of `box`. Each `box` is a + [`box_coord`, `box_labels`] where `box_coord` is the + coordinates of box and 'box_labels` are the corresponding + labels for the box. + """ + gt_lists = cfg.AVA.TRAIN_GT_BOX_LISTS if mode == "train" else [] + pred_lists = ( + cfg.AVA.TRAIN_PREDICT_BOX_LISTS + if mode == "train" + else cfg.AVA.TEST_PREDICT_BOX_LISTS + ) + ann_filenames = [ + os.path.join(cfg.AVA.ANNOTATION_DIR, filename) + for filename in gt_lists + pred_lists + ] + ann_is_gt_box = [True] * len(gt_lists) + [False] * len(pred_lists) + + detect_thresh = cfg.AVA.DETECTION_SCORE_THRESH + # Only select frame_sec % 4 = 0 samples for validation if not + # set FULL_TEST_ON_VAL. + boxes_sample_rate = ( + 4 if mode == "val" and not cfg.AVA.FULL_TEST_ON_VAL else 1 + ) + all_boxes, count, unique_box_count = parse_bboxes_file( + ann_filenames=ann_filenames, + ann_is_gt_box=ann_is_gt_box, + detect_thresh=detect_thresh, + boxes_sample_rate=boxes_sample_rate, + ) + logger.info( + "Finished loading annotations from: %s" % ", ".join(ann_filenames) + ) + logger.info("Detection threshold: {}".format(detect_thresh)) + logger.info("Number of unique boxes: %d" % unique_box_count) + logger.info("Number of annotations: %d" % count) + + return all_boxes + + +def get_keyframe_data(boxes_and_labels): + """ + Getting keyframe indices, boxes and labels in the dataset. + + Args: + boxes_and_labels (list[dict]): a list which maps from video_idx to a dict. + Each dict `frame_sec` to a list of boxes and corresponding labels. + + Returns: + keyframe_indices (list): a list of indices of the keyframes. + keyframe_boxes_and_labels (list[list[list]]): a list of list which maps from + video_idx and sec_idx to a list of boxes and corresponding labels. + """ + + def sec_to_frame(sec): + """ + Convert time index (in second) to frame index. + 0: 900 + 30: 901 + """ + return (sec - 900) * FPS + + keyframe_indices = [] + keyframe_boxes_and_labels = [] + count = 0 + for video_idx in range(len(boxes_and_labels)): + sec_idx = 0 + keyframe_boxes_and_labels.append([]) + for sec in boxes_and_labels[video_idx].keys(): + if sec not in AVA_VALID_FRAMES: + continue + + if len(boxes_and_labels[video_idx][sec]) > 0: + keyframe_indices.append( + (video_idx, sec_idx, sec, sec_to_frame(sec)) + ) + keyframe_boxes_and_labels[video_idx].append( + boxes_and_labels[video_idx][sec] + ) + sec_idx += 1 + count += 1 + logger.info("%d keyframes used." % count) + + return keyframe_indices, keyframe_boxes_and_labels + + +def get_num_boxes_used(keyframe_indices, keyframe_boxes_and_labels): + """ + Get total number of used boxes. + + Args: + keyframe_indices (list): a list of indices of the keyframes. + keyframe_boxes_and_labels (list[list[list]]): a list of list which maps from + video_idx and sec_idx to a list of boxes and corresponding labels. + + Returns: + count (int): total number of used boxes. + """ + + count = 0 + for video_idx, sec_idx, _, _ in keyframe_indices: + count += len(keyframe_boxes_and_labels[video_idx][sec_idx]) + return count + + +def parse_bboxes_file( + ann_filenames, ann_is_gt_box, detect_thresh, boxes_sample_rate=1 +): + """ + Parse AVA bounding boxes files. + Args: + ann_filenames (list of str(s)): a list of AVA bounding boxes annotation files. + ann_is_gt_box (list of bools): a list of boolean to indicate whether the corresponding + ann_file is ground-truth. `ann_is_gt_box[i]` correspond to `ann_filenames[i]`. + detect_thresh (float): threshold for accepting predicted boxes, range [0, 1]. + boxes_sample_rate (int): sample rate for test bounding boxes. Get 1 every `boxes_sample_rate`. + """ + all_boxes = {} + count = 0 + unique_box_count = 0 + for filename, is_gt_box in zip(ann_filenames, ann_is_gt_box): + with g_pathmgr.open(filename, "r") as f: + for line in f: + row = line.strip().split(",") + # When we use predicted boxes to train/eval, we need to + # ignore the boxes whose scores are below the threshold. + if not is_gt_box: + score = float(row[7]) + if score < detect_thresh: + continue + + video_name, frame_sec = row[0], int(row[1]) + if frame_sec % boxes_sample_rate != 0: + continue + + # Box with format [x1, y1, x2, y2] with a range of [0, 1] as float. + box_key = ",".join(row[2:6]) + box = list(map(float, row[2:6])) + label = -1 if row[6] == "" else int(row[6]) + + if video_name not in all_boxes: + all_boxes[video_name] = {} + for sec in AVA_VALID_FRAMES: + all_boxes[video_name][sec] = {} + + if box_key not in all_boxes[video_name][frame_sec]: + all_boxes[video_name][frame_sec][box_key] = [box, []] + unique_box_count += 1 + + all_boxes[video_name][frame_sec][box_key][1].append(label) + if label != -1: + count += 1 + + for video_name in all_boxes.keys(): + for frame_sec in all_boxes[video_name].keys(): + # Save in format of a list of [box_i, box_i_labels]. + all_boxes[video_name][frame_sec] = list( + all_boxes[video_name][frame_sec].values() + ) + + return all_boxes, count, unique_box_count diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/build.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/build.py" new file mode 100644 index 0000000..323f4e8 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/build.py" @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +from fvcore.common.registry import Registry + +DATASET_REGISTRY = Registry("DATASET") +DATASET_REGISTRY.__doc__ = """ +Registry for dataset. + +The registered object will be called with `obj(cfg, split)`. +The call should return a `torch.utils.data.Dataset` object. +""" + + +def build_dataset(dataset_name, cfg, split): + """ + Build a dataset, defined by `dataset_name`. + Args: + dataset_name (str): the name of the dataset to be constructed. + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + split (str): the split of the data loader. Options include `train`, + `val`, and `test`. + Returns: + Dataset: a constructed dataset specified by dataset_name. + """ + # Capitalize the the first letter of the dataset_name since the dataset_name + # in configs may be in lowercase but the name of dataset class should always + # start with an uppercase letter. + name = dataset_name.capitalize() + return DATASET_REGISTRY.get(name)(cfg, split) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/charades.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/charades.py" new file mode 100644 index 0000000..39ee65f --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/charades.py" @@ -0,0 +1,258 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import os +import random +from itertools import chain as chain +import torch +import torch.utils.data +from iopath.common.file_io import g_pathmgr + +import slowfast.utils.logging as logging + +from . import utils as utils +from .build import DATASET_REGISTRY + +logger = logging.get_logger(__name__) + + +@DATASET_REGISTRY.register() +class Charades(torch.utils.data.Dataset): + """ + Charades video loader. Construct the Charades video loader, then sample + clips from the videos. For training and validation, a single clip is randomly + sampled from every video with random cropping, scaling, and flipping. For + testing, multiple clips are uniformaly sampled from every video with uniform + cropping. For uniform cropping, we take the left, center, and right crop if + the width is larger than height, or take top, center, and bottom crop if the + height is larger than the width. + """ + + def __init__(self, cfg, mode, num_retries=10): + """ + Load Charades data (frame paths, labels, etc. ) to a given Dataset object. + The dataset could be downloaded from Chrades official website + (https://allenai.org/plato/charades/). + Please see datasets/DATASET.md for more information about the data format. + Args: + dataset (Dataset): a Dataset object to load Charades data to. + mode (string): 'train', 'val', or 'test'. + Args: + cfg (CfgNode): configs. + mode (string): Options includes `train`, `val`, or `test` mode. + For the train and val mode, the data loader will take data + from the train or val set, and sample one clip per video. + For the test mode, the data loader will take data from test set, + and sample multiple clips per video. + num_retries (int): number of retries. + """ + # Only support train, val, and test mode. + assert mode in [ + "train", + "val", + "test", + ], "Split '{}' not supported for Charades ".format(mode) + self.mode = mode + self.cfg = cfg + + self._video_meta = {} + self._num_retries = num_retries + # For training or validation mode, one single clip is sampled from every + # video. For testing, NUM_ENSEMBLE_VIEWS clips are sampled from every + # video. For every clip, NUM_SPATIAL_CROPS is cropped spatially from + # the frames. + if self.mode in ["train", "val"]: + self._num_clips = 1 + elif self.mode in ["test"]: + self._num_clips = ( + cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS + ) + + logger.info("Constructing Charades {}...".format(mode)) + self._construct_loader() + + def _construct_loader(self): + """ + Construct the video loader. + """ + path_to_file = os.path.join( + self.cfg.DATA.PATH_TO_DATA_DIR, + "{}.csv".format("train" if self.mode == "train" else "val"), + ) + assert g_pathmgr.exists(path_to_file), "{} dir not found".format( + path_to_file + ) + (self._path_to_videos, self._labels) = utils.load_image_lists( + path_to_file, self.cfg.DATA.PATH_PREFIX, return_list=True + ) + + if self.mode != "train": + # Form video-level labels from frame level annotations. + self._labels = utils.convert_to_video_level_labels(self._labels) + + self._path_to_videos = list( + chain.from_iterable( + [[x] * self._num_clips for x in self._path_to_videos] + ) + ) + self._labels = list( + chain.from_iterable([[x] * self._num_clips for x in self._labels]) + ) + self._spatial_temporal_idx = list( + chain.from_iterable( + [range(self._num_clips) for _ in range(len(self._labels))] + ) + ) + + logger.info( + "Charades dataloader constructed (size: {}) from {}".format( + len(self._path_to_videos), path_to_file + ) + ) + + def get_seq_frames(self, index): + """ + Given the video index, return the list of indexs of sampled frames. + Args: + index (int): the video index. + Returns: + seq (list): the indexes of sampled frames from the video. + """ + temporal_sample_index = ( + -1 + if self.mode in ["train", "val"] + else self._spatial_temporal_idx[index] + // self.cfg.TEST.NUM_SPATIAL_CROPS + ) + num_frames = self.cfg.DATA.NUM_FRAMES + sampling_rate = utils.get_random_sampling_rate( + self.cfg.MULTIGRID.LONG_CYCLE_SAMPLING_RATE, + self.cfg.DATA.SAMPLING_RATE, + ) + video_length = len(self._path_to_videos[index]) + assert video_length == len(self._labels[index]) + + clip_length = (num_frames - 1) * sampling_rate + 1 + if temporal_sample_index == -1: + if clip_length > video_length: + start = random.randint(video_length - clip_length, 0) + else: + start = random.randint(0, video_length - clip_length) + else: + gap = float(max(video_length - clip_length, 0)) / ( + self.cfg.TEST.NUM_ENSEMBLE_VIEWS - 1 + ) + start = int(round(gap * temporal_sample_index)) + + seq = [ + max(min(start + i * sampling_rate, video_length - 1), 0) + for i in range(num_frames) + ] + + return seq + + def __getitem__(self, index): + """ + Given the video index, return the list of frames, label, and video + index if the video frames can be fetched. + Args: + index (int): the video index provided by the pytorch sampler. + Returns: + frames (tensor): the frames of sampled from the video. The dimension + is `channel` x `num frames` x `height` x `width`. + label (int): the label of the current video. + index (int): the index of the video. + """ + short_cycle_idx = None + # When short cycle is used, input index is a tupple. + if isinstance(index, tuple): + index, short_cycle_idx = index + + if self.mode in ["train", "val"]: + # -1 indicates random sampling. + spatial_sample_index = -1 + min_scale = self.cfg.DATA.TRAIN_JITTER_SCALES[0] + max_scale = self.cfg.DATA.TRAIN_JITTER_SCALES[1] + crop_size = self.cfg.DATA.TRAIN_CROP_SIZE + if short_cycle_idx in [0, 1]: + crop_size = int( + round( + self.cfg.MULTIGRID.SHORT_CYCLE_FACTORS[short_cycle_idx] + * self.cfg.MULTIGRID.DEFAULT_S + ) + ) + if self.cfg.MULTIGRID.DEFAULT_S > 0: + # Decreasing the scale is equivalent to using a larger "span" + # in a sampling grid. + min_scale = int( + round( + float(min_scale) + * crop_size + / self.cfg.MULTIGRID.DEFAULT_S + ) + ) + elif self.mode in ["test"]: + # spatial_sample_index is in [0, 1, 2]. Corresponding to left, + # center, or right if width is larger than height, and top, middle, + # or bottom if height is larger than width. + spatial_sample_index = ( + self._spatial_temporal_idx[index] + % self.cfg.TEST.NUM_SPATIAL_CROPS + ) + min_scale, max_scale, crop_size = [self.cfg.DATA.TEST_CROP_SIZE] * 3 + # The testing is deterministic and no jitter should be performed. + # min_scale, max_scale, and crop_size are expect to be the same. + assert len({min_scale, max_scale, crop_size}) == 1 + else: + raise NotImplementedError( + "Does not support {} mode".format(self.mode) + ) + + seq = self.get_seq_frames(index) + frames = torch.as_tensor( + utils.retry_load_images( + [self._path_to_videos[index][frame] for frame in seq], + self._num_retries, + ) + ) + + label = utils.aggregate_labels( + [self._labels[index][i] for i in range(seq[0], seq[-1] + 1)] + ) + label = torch.as_tensor( + utils.as_binary_vector(label, self.cfg.MODEL.NUM_CLASSES) + ) + + # Perform color normalization. + frames = utils.tensor_normalize( + frames, self.cfg.DATA.MEAN, self.cfg.DATA.STD + ) + # T H W C -> C T H W. + frames = frames.permute(3, 0, 1, 2) + # Perform data augmentation. + frames = utils.spatial_sampling( + frames, + spatial_idx=spatial_sample_index, + min_scale=min_scale, + max_scale=max_scale, + crop_size=crop_size, + random_horizontal_flip=self.cfg.DATA.RANDOM_FLIP, + inverse_uniform_sampling=self.cfg.DATA.INV_UNIFORM_SAMPLE, + ) + frames = utils.pack_pathway_output(self.cfg, frames) + return frames, label, index, {} + + def __len__(self): + """ + Returns: + (int): the number of videos in the dataset. + """ + return self.num_videos + + @property + def num_videos(self): + """ + Returns: + (int): the number of videos in the dataset. + """ + return len(self._path_to_videos) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/cv2_transform.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/cv2_transform.py" new file mode 100644 index 0000000..e375e50 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/cv2_transform.py" @@ -0,0 +1,797 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import math +import numpy as np +import cv2 + + +def clip_boxes_to_image(boxes, height, width): + """ + Clip the boxes with the height and width of the image size. + Args: + boxes (ndarray): bounding boxes to peform crop. The dimension is + `num boxes` x 4. + height (int): the height of the image. + width (int): the width of the image. + Returns: + boxes (ndarray): cropped bounding boxes. + """ + boxes[:, [0, 2]] = np.minimum( + width - 1.0, np.maximum(0.0, boxes[:, [0, 2]]) + ) + boxes[:, [1, 3]] = np.minimum( + height - 1.0, np.maximum(0.0, boxes[:, [1, 3]]) + ) + return boxes + + +def random_short_side_scale_jitter_list(images, min_size, max_size, boxes=None): + """ + Perform a spatial short scale jittering on the given images and + corresponding boxes. + Args: + images (list): list of images to perform scale jitter. Dimension is + `height` x `width` x `channel`. + min_size (int): the minimal size to scale the frames. + max_size (int): the maximal size to scale the frames. + boxes (list): optional. Corresponding boxes to images. Dimension is + `num boxes` x 4. + Returns: + (list): the list of scaled images with dimension of + `new height` x `new width` x `channel`. + (ndarray or None): the scaled boxes with dimension of + `num boxes` x 4. + """ + size = int(round(1.0 / np.random.uniform(1.0 / max_size, 1.0 / min_size))) + + height = images[0].shape[0] + width = images[0].shape[1] + if (width <= height and width == size) or ( + height <= width and height == size + ): + return images, boxes + new_width = size + new_height = size + if width < height: + new_height = int(math.floor((float(height) / width) * size)) + if boxes is not None: + boxes = [ + proposal * float(new_height) / height for proposal in boxes + ] + else: + new_width = int(math.floor((float(width) / height) * size)) + if boxes is not None: + boxes = [proposal * float(new_width) / width for proposal in boxes] + return ( + [ + cv2.resize( + image, (new_width, new_height), interpolation=cv2.INTER_LINEAR + ).astype(np.float32) + for image in images + ], + boxes, + ) + + +def scale(size, image): + """ + Scale the short side of the image to size. + Args: + size (int): size to scale the image. + image (array): image to perform short side scale. Dimension is + `height` x `width` x `channel`. + Returns: + (ndarray): the scaled image with dimension of + `height` x `width` x `channel`. + """ + height = image.shape[0] + width = image.shape[1] + if (width <= height and width == size) or ( + height <= width and height == size + ): + return image + new_width = size + new_height = size + if width < height: + new_height = int(math.floor((float(height) / width) * size)) + else: + new_width = int(math.floor((float(width) / height) * size)) + img = cv2.resize( + image, (new_width, new_height), interpolation=cv2.INTER_LINEAR + ) + return img.astype(np.float32) + + +def scale_boxes(size, boxes, height, width): + """ + Scale the short side of the box to size. + Args: + size (int): size to scale the image. + boxes (ndarray): bounding boxes to peform scale. The dimension is + `num boxes` x 4. + height (int): the height of the image. + width (int): the width of the image. + Returns: + boxes (ndarray): scaled bounding boxes. + """ + if (width <= height and width == size) or ( + height <= width and height == size + ): + return boxes + + new_width = size + new_height = size + if width < height: + new_height = int(math.floor((float(height) / width) * size)) + boxes *= float(new_height) / height + else: + new_width = int(math.floor((float(width) / height) * size)) + boxes *= float(new_width) / width + return boxes + + +def horizontal_flip_list(prob, images, order="CHW", boxes=None): + """ + Horizontally flip the list of image and optional boxes. + Args: + prob (float): probability to flip. + image (list): ilist of images to perform short side scale. Dimension is + `height` x `width` x `channel` or `channel` x `height` x `width`. + order (str): order of the `height`, `channel` and `width`. + boxes (list): optional. Corresponding boxes to images. + Dimension is `num boxes` x 4. + Returns: + (ndarray): the scaled image with dimension of + `height` x `width` x `channel`. + (list): optional. Corresponding boxes to images. Dimension is + `num boxes` x 4. + """ + _, width, _ = images[0].shape + if np.random.uniform() < prob: + if boxes is not None: + boxes = [flip_boxes(proposal, width) for proposal in boxes] + if order == "CHW": + out_images = [] + for image in images: + image = np.asarray(image).swapaxes(2, 0) + image = image[::-1] + out_images.append(image.swapaxes(0, 2)) + return out_images, boxes + elif order == "HWC": + return [cv2.flip(image, 1) for image in images], boxes + return images, boxes + + +def spatial_shift_crop_list(size, images, spatial_shift_pos, boxes=None): + """ + Perform left, center, or right crop of the given list of images. + Args: + size (int): size to crop. + image (list): ilist of images to perform short side scale. Dimension is + `height` x `width` x `channel` or `channel` x `height` x `width`. + spatial_shift_pos (int): option includes 0 (left), 1 (middle), and + 2 (right) crop. + boxes (list): optional. Corresponding boxes to images. + Dimension is `num boxes` x 4. + Returns: + cropped (ndarray): the cropped list of images with dimension of + `height` x `width` x `channel`. + boxes (list): optional. Corresponding boxes to images. Dimension is + `num boxes` x 4. + """ + + assert spatial_shift_pos in [0, 1, 2] + + height = images[0].shape[0] + width = images[0].shape[1] + y_offset = int(math.ceil((height - size) / 2)) + x_offset = int(math.ceil((width - size) / 2)) + + if height > width: + if spatial_shift_pos == 0: + y_offset = 0 + elif spatial_shift_pos == 2: + y_offset = height - size + else: + if spatial_shift_pos == 0: + x_offset = 0 + elif spatial_shift_pos == 2: + x_offset = width - size + + cropped = [ + image[y_offset : y_offset + size, x_offset : x_offset + size, :] + for image in images + ] + assert cropped[0].shape[0] == size, "Image height not cropped properly" + assert cropped[0].shape[1] == size, "Image width not cropped properly" + + if boxes is not None: + for i in range(len(boxes)): + boxes[i][:, [0, 2]] -= x_offset + boxes[i][:, [1, 3]] -= y_offset + return cropped, boxes + + +def CHW2HWC(image): + """ + Transpose the dimension from `channel` x `height` x `width` to + `height` x `width` x `channel`. + Args: + image (array): image to transpose. + Returns + (array): transposed image. + """ + return image.transpose([1, 2, 0]) + + +def HWC2CHW(image): + """ + Transpose the dimension from `height` x `width` x `channel` to + `channel` x `height` x `width`. + Args: + image (array): image to transpose. + Returns + (array): transposed image. + """ + return image.transpose([2, 0, 1]) + + +def color_jitter_list( + images, img_brightness=0, img_contrast=0, img_saturation=0 +): + """ + Perform color jitter on the list of images. + Args: + images (list): list of images to perform color jitter. + img_brightness (float): jitter ratio for brightness. + img_contrast (float): jitter ratio for contrast. + img_saturation (float): jitter ratio for saturation. + Returns: + images (list): the jittered list of images. + """ + jitter = [] + if img_brightness != 0: + jitter.append("brightness") + if img_contrast != 0: + jitter.append("contrast") + if img_saturation != 0: + jitter.append("saturation") + + if len(jitter) > 0: + order = np.random.permutation(np.arange(len(jitter))) + for idx in range(0, len(jitter)): + if jitter[order[idx]] == "brightness": + images = brightness_list(img_brightness, images) + elif jitter[order[idx]] == "contrast": + images = contrast_list(img_contrast, images) + elif jitter[order[idx]] == "saturation": + images = saturation_list(img_saturation, images) + return images + + +def lighting_list(imgs, alphastd, eigval, eigvec, alpha=None): + """ + Perform AlexNet-style PCA jitter on the given list of images. + Args: + images (list): list of images to perform lighting jitter. + alphastd (float): jitter ratio for PCA jitter. + eigval (list): eigenvalues for PCA jitter. + eigvec (list[list]): eigenvectors for PCA jitter. + Returns: + out_images (list): the list of jittered images. + """ + if alphastd == 0: + return imgs + # generate alpha1, alpha2, alpha3 + alpha = np.random.normal(0, alphastd, size=(1, 3)) + eig_vec = np.array(eigvec) + eig_val = np.reshape(eigval, (1, 3)) + rgb = np.sum( + eig_vec * np.repeat(alpha, 3, axis=0) * np.repeat(eig_val, 3, axis=0), + axis=1, + ) + out_images = [] + for img in imgs: + for idx in range(img.shape[0]): + img[idx] = img[idx] + rgb[2 - idx] + out_images.append(img) + return out_images + + +def color_normalization(image, mean, stddev): + """ + Perform color normalization on the image with the given mean and stddev. + Args: + image (array): image to perform color normalization. + mean (float): mean value to subtract. + stddev (float): stddev to devide. + """ + # Input image should in format of CHW + assert len(mean) == image.shape[0], "channel mean not computed properly" + assert len(stddev) == image.shape[0], "channel stddev not computed properly" + for idx in range(image.shape[0]): + image[idx] = image[idx] - mean[idx] + image[idx] = image[idx] / stddev[idx] + return image + + +def pad_image(image, pad_size, order="CHW"): + """ + Pad the given image with the size of pad_size. + Args: + image (array): image to pad. + pad_size (int): size to pad. + order (str): order of the `height`, `channel` and `width`. + Returns: + img (array): padded image. + """ + if order == "CHW": + img = np.pad( + image, + ((0, 0), (pad_size, pad_size), (pad_size, pad_size)), + mode=str("constant"), + ) + elif order == "HWC": + img = np.pad( + image, + ((pad_size, pad_size), (pad_size, pad_size), (0, 0)), + mode=str("constant"), + ) + return img + + +def horizontal_flip(prob, image, order="CHW"): + """ + Horizontally flip the image. + Args: + prob (float): probability to flip. + image (array): image to pad. + order (str): order of the `height`, `channel` and `width`. + Returns: + img (array): flipped image. + """ + assert order in ["CHW", "HWC"], "order {} is not supported".format(order) + if np.random.uniform() < prob: + if order == "CHW": + image = image[:, :, ::-1] + elif order == "HWC": + image = image[:, ::-1, :] + else: + raise NotImplementedError("Unknown order {}".format(order)) + return image + + +def flip_boxes(boxes, im_width): + """ + Horizontally flip the boxes. + Args: + boxes (array): box to flip. + im_width (int): width of the image. + Returns: + boxes_flipped (array): flipped box. + """ + + boxes_flipped = boxes.copy() + boxes_flipped[:, 0::4] = im_width - boxes[:, 2::4] - 1 + boxes_flipped[:, 2::4] = im_width - boxes[:, 0::4] - 1 + return boxes_flipped + + +def crop_boxes(boxes, x_offset, y_offset): + """ + Crop the boxes given the offsets. + Args: + boxes (array): boxes to crop. + x_offset (int): offset on x. + y_offset (int): offset on y. + """ + boxes[:, [0, 2]] = boxes[:, [0, 2]] - x_offset + boxes[:, [1, 3]] = boxes[:, [1, 3]] - y_offset + return boxes + + +def random_crop_list(images, size, pad_size=0, order="CHW", boxes=None): + """ + Perform random crop on a list of images. + Args: + images (list): list of images to perform random crop. + size (int): size to crop. + pad_size (int): padding size. + order (str): order of the `height`, `channel` and `width`. + boxes (list): optional. Corresponding boxes to images. + Dimension is `num boxes` x 4. + Returns: + cropped (ndarray): the cropped list of images with dimension of + `height` x `width` x `channel`. + boxes (list): optional. Corresponding boxes to images. Dimension is + `num boxes` x 4. + """ + # explicitly dealing processing per image order to avoid flipping images. + if pad_size > 0: + images = [ + pad_image(pad_size=pad_size, image=image, order=order) + for image in images + ] + + # image format should be CHW. + if order == "CHW": + if images[0].shape[1] == size and images[0].shape[2] == size: + return images, boxes + height = images[0].shape[1] + width = images[0].shape[2] + y_offset = 0 + if height > size: + y_offset = int(np.random.randint(0, height - size)) + x_offset = 0 + if width > size: + x_offset = int(np.random.randint(0, width - size)) + cropped = [ + image[:, y_offset : y_offset + size, x_offset : x_offset + size] + for image in images + ] + assert cropped[0].shape[1] == size, "Image not cropped properly" + assert cropped[0].shape[2] == size, "Image not cropped properly" + elif order == "HWC": + if images[0].shape[0] == size and images[0].shape[1] == size: + return images, boxes + height = images[0].shape[0] + width = images[0].shape[1] + y_offset = 0 + if height > size: + y_offset = int(np.random.randint(0, height - size)) + x_offset = 0 + if width > size: + x_offset = int(np.random.randint(0, width - size)) + cropped = [ + image[y_offset : y_offset + size, x_offset : x_offset + size, :] + for image in images + ] + assert cropped[0].shape[0] == size, "Image not cropped properly" + assert cropped[0].shape[1] == size, "Image not cropped properly" + + if boxes is not None: + boxes = [crop_boxes(proposal, x_offset, y_offset) for proposal in boxes] + return cropped, boxes + + +def center_crop(size, image): + """ + Perform center crop on input images. + Args: + size (int): size of the cropped height and width. + image (array): the image to perform center crop. + """ + height = image.shape[0] + width = image.shape[1] + y_offset = int(math.ceil((height - size) / 2)) + x_offset = int(math.ceil((width - size) / 2)) + cropped = image[y_offset : y_offset + size, x_offset : x_offset + size, :] + assert cropped.shape[0] == size, "Image height not cropped properly" + assert cropped.shape[1] == size, "Image width not cropped properly" + return cropped + + +# ResNet style scale jittering: randomly select the scale from +# [1/max_size, 1/min_size] +def random_scale_jitter(image, min_size, max_size): + """ + Perform ResNet style random scale jittering: randomly select the scale from + [1/max_size, 1/min_size]. + Args: + image (array): image to perform random scale. + min_size (int): min size to scale. + max_size (int) max size to scale. + Returns: + image (array): scaled image. + """ + img_scale = int( + round(1.0 / np.random.uniform(1.0 / max_size, 1.0 / min_size)) + ) + image = scale(img_scale, image) + return image + + +def random_scale_jitter_list(images, min_size, max_size): + """ + Perform ResNet style random scale jittering on a list of image: randomly + select the scale from [1/max_size, 1/min_size]. Note that all the image + will share the same scale. + Args: + images (list): list of images to perform random scale. + min_size (int): min size to scale. + max_size (int) max size to scale. + Returns: + images (list): list of scaled image. + """ + img_scale = int( + round(1.0 / np.random.uniform(1.0 / max_size, 1.0 / min_size)) + ) + return [scale(img_scale, image) for image in images] + + +def random_sized_crop(image, size, area_frac=0.08): + """ + Perform random sized cropping on the given image. Random crop with size + 8% - 100% image area and aspect ratio in [3/4, 4/3]. + Args: + image (array): image to crop. + size (int): size to crop. + area_frac (float): area of fraction. + Returns: + (array): cropped image. + """ + for _ in range(0, 10): + height = image.shape[0] + width = image.shape[1] + area = height * width + target_area = np.random.uniform(area_frac, 1.0) * area + aspect_ratio = np.random.uniform(3.0 / 4.0, 4.0 / 3.0) + w = int(round(math.sqrt(float(target_area) * aspect_ratio))) + h = int(round(math.sqrt(float(target_area) / aspect_ratio))) + if np.random.uniform() < 0.5: + w, h = h, w + if h <= height and w <= width: + if height == h: + y_offset = 0 + else: + y_offset = np.random.randint(0, height - h) + if width == w: + x_offset = 0 + else: + x_offset = np.random.randint(0, width - w) + y_offset = int(y_offset) + x_offset = int(x_offset) + cropped = image[y_offset : y_offset + h, x_offset : x_offset + w, :] + assert ( + cropped.shape[0] == h and cropped.shape[1] == w + ), "Wrong crop size" + cropped = cv2.resize( + cropped, (size, size), interpolation=cv2.INTER_LINEAR + ) + return cropped.astype(np.float32) + return center_crop(size, scale(size, image)) + + +def lighting(img, alphastd, eigval, eigvec): + """ + Perform AlexNet-style PCA jitter on the given image. + Args: + image (array): list of images to perform lighting jitter. + alphastd (float): jitter ratio for PCA jitter. + eigval (array): eigenvalues for PCA jitter. + eigvec (list): eigenvectors for PCA jitter. + Returns: + img (tensor): the jittered image. + """ + if alphastd == 0: + return img + # generate alpha1, alpha2, alpha3. + alpha = np.random.normal(0, alphastd, size=(1, 3)) + eig_vec = np.array(eigvec) + eig_val = np.reshape(eigval, (1, 3)) + rgb = np.sum( + eig_vec * np.repeat(alpha, 3, axis=0) * np.repeat(eig_val, 3, axis=0), + axis=1, + ) + for idx in range(img.shape[0]): + img[idx] = img[idx] + rgb[2 - idx] + return img + + +def random_sized_crop_list(images, size, crop_area_fraction=0.08): + """ + Perform random sized cropping on the given list of images. Random crop with + size 8% - 100% image area and aspect ratio in [3/4, 4/3]. + Args: + images (list): image to crop. + size (int): size to crop. + area_frac (float): area of fraction. + Returns: + (list): list of cropped image. + """ + for _ in range(0, 10): + height = images[0].shape[0] + width = images[0].shape[1] + area = height * width + target_area = np.random.uniform(crop_area_fraction, 1.0) * area + aspect_ratio = np.random.uniform(3.0 / 4.0, 4.0 / 3.0) + w = int(round(math.sqrt(float(target_area) * aspect_ratio))) + h = int(round(math.sqrt(float(target_area) / aspect_ratio))) + if np.random.uniform() < 0.5: + w, h = h, w + if h <= height and w <= width: + if height == h: + y_offset = 0 + else: + y_offset = np.random.randint(0, height - h) + if width == w: + x_offset = 0 + else: + x_offset = np.random.randint(0, width - w) + y_offset = int(y_offset) + x_offset = int(x_offset) + + croppsed_images = [] + for image in images: + cropped = image[ + y_offset : y_offset + h, x_offset : x_offset + w, : + ] + assert ( + cropped.shape[0] == h and cropped.shape[1] == w + ), "Wrong crop size" + cropped = cv2.resize( + cropped, (size, size), interpolation=cv2.INTER_LINEAR + ) + croppsed_images.append(cropped.astype(np.float32)) + return croppsed_images + + return [center_crop(size, scale(size, image)) for image in images] + + +def blend(image1, image2, alpha): + return image1 * alpha + image2 * (1 - alpha) + + +def grayscale(image): + """ + Convert the image to gray scale. + Args: + image (tensor): image to convert to gray scale. Dimension is + `channel` x `height` x `width`. + Returns: + img_gray (tensor): image in gray scale. + """ + # R -> 0.299, G -> 0.587, B -> 0.114. + img_gray = np.copy(image) + gray_channel = 0.299 * image[2] + 0.587 * image[1] + 0.114 * image[0] + img_gray[0] = gray_channel + img_gray[1] = gray_channel + img_gray[2] = gray_channel + return img_gray + + +def saturation(var, image): + """ + Perform color saturation on the given image. + Args: + var (float): variance. + image (array): image to perform color saturation. + Returns: + (array): image that performed color saturation. + """ + img_gray = grayscale(image) + alpha = 1.0 + np.random.uniform(-var, var) + return blend(image, img_gray, alpha) + + +def brightness(var, image): + """ + Perform color brightness on the given image. + Args: + var (float): variance. + image (array): image to perform color brightness. + Returns: + (array): image that performed color brightness. + """ + img_bright = np.zeros(image.shape).astype(image.dtype) + alpha = 1.0 + np.random.uniform(-var, var) + return blend(image, img_bright, alpha) + + +def contrast(var, image): + """ + Perform color contrast on the given image. + Args: + var (float): variance. + image (array): image to perform color contrast. + Returns: + (array): image that performed color contrast. + """ + img_gray = grayscale(image) + img_gray.fill(np.mean(img_gray[0])) + alpha = 1.0 + np.random.uniform(-var, var) + return blend(image, img_gray, alpha) + + +def saturation_list(var, images): + """ + Perform color saturation on the list of given images. + Args: + var (float): variance. + images (list): list of images to perform color saturation. + Returns: + (list): list of images that performed color saturation. + """ + alpha = 1.0 + np.random.uniform(-var, var) + + out_images = [] + for image in images: + img_gray = grayscale(image) + out_images.append(blend(image, img_gray, alpha)) + return out_images + + +def brightness_list(var, images): + """ + Perform color brightness on the given list of images. + Args: + var (float): variance. + images (list): list of images to perform color brightness. + Returns: + (array): list of images that performed color brightness. + """ + alpha = 1.0 + np.random.uniform(-var, var) + + out_images = [] + for image in images: + img_bright = np.zeros(image.shape).astype(image.dtype) + out_images.append(blend(image, img_bright, alpha)) + return out_images + + +def contrast_list(var, images): + """ + Perform color contrast on the given list of images. + Args: + var (float): variance. + images (list): list of images to perform color contrast. + Returns: + (array): image that performed color contrast. + """ + alpha = 1.0 + np.random.uniform(-var, var) + + out_images = [] + for image in images: + img_gray = grayscale(image) + img_gray.fill(np.mean(img_gray[0])) + out_images.append(blend(image, img_gray, alpha)) + return out_images + + +def color_jitter(image, img_brightness=0, img_contrast=0, img_saturation=0): + """ + Perform color jitter on the given image. + Args: + image (array): image to perform color jitter. + img_brightness (float): jitter ratio for brightness. + img_contrast (float): jitter ratio for contrast. + img_saturation (float): jitter ratio for saturation. + Returns: + image (array): the jittered image. + """ + jitter = [] + if img_brightness != 0: + jitter.append("brightness") + if img_contrast != 0: + jitter.append("contrast") + if img_saturation != 0: + jitter.append("saturation") + + if len(jitter) > 0: + order = np.random.permutation(np.arange(len(jitter))) + for idx in range(0, len(jitter)): + if jitter[order[idx]] == "brightness": + image = brightness(img_brightness, image) + elif jitter[order[idx]] == "contrast": + image = contrast(img_contrast, image) + elif jitter[order[idx]] == "saturation": + image = saturation(img_saturation, image) + return image + + +def revert_scaled_boxes(size, boxes, img_height, img_width): + """ + Revert scaled input boxes to match the original image size. + Args: + size (int): size of the cropped image. + boxes (array): shape (num_boxes, 4). + img_height (int): height of original image. + img_width (int): width of original image. + Returns: + reverted_boxes (array): boxes scaled back to the original image size. + """ + scaled_aspect = np.min([img_height, img_width]) + scale_ratio = scaled_aspect / size + reverted_boxes = boxes * scale_ratio + return reverted_boxes diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/decoder.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/decoder.py" new file mode 100644 index 0000000..a101bff --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/decoder.py" @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import math +import numpy as np +import random +import torch +import torchvision.io as io + + +def temporal_sampling(frames, start_idx, end_idx, num_samples): + """ + Given the start and end frame index, sample num_samples frames between + the start and end with equal interval. + Args: + frames (tensor): a tensor of video frames, dimension is + `num video frames` x `channel` x `height` x `width`. + start_idx (int): the index of the start frame. + end_idx (int): the index of the end frame. + num_samples (int): number of frames to sample. + Returns: + frames (tersor): a tensor of temporal sampled video frames, dimension is + `num clip frames` x `channel` x `height` x `width`. + """ + index = torch.linspace(start_idx, end_idx, num_samples) + index = torch.clamp(index, 0, frames.shape[0] - 1).long() + frames = torch.index_select(frames, 0, index) + return frames + + +def get_start_end_idx( + video_size, clip_size, clip_idx, num_clips, use_offset=False +): + """ + Sample a clip of size clip_size from a video of size video_size and + return the indices of the first and last frame of the clip. If clip_idx is + -1, the clip is randomly sampled, otherwise uniformly split the video to + num_clips clips, and select the start and end index of clip_idx-th video + clip. + Args: + video_size (int): number of overall frames. + clip_size (int): size of the clip to sample from the frames. + clip_idx (int): if clip_idx is -1, perform random jitter sampling. If + clip_idx is larger than -1, uniformly split the video to num_clips + clips, and select the start and end index of the clip_idx-th video + clip. + num_clips (int): overall number of clips to uniformly sample from the + given video for testing. + Returns: + start_idx (int): the start frame index. + end_idx (int): the end frame index. + """ + delta = max(video_size - clip_size, 0) + if clip_idx == -1: + # Random temporal sampling. + start_idx = random.uniform(0, delta) + else: + if use_offset: + if num_clips == 1: + # Take the center clip if num_clips is 1. + start_idx = math.floor(delta / 2) + else: + # Uniformly sample the clip with the given index. + start_idx = clip_idx * math.floor(delta / (num_clips - 1)) + else: + # Uniformly sample the clip with the given index. + start_idx = delta * clip_idx / num_clips + end_idx = start_idx + clip_size - 1 + return start_idx, end_idx + + +def pyav_decode_stream( + container, start_pts, end_pts, stream, stream_name, buffer_size=0 +): + """ + Decode the video with PyAV decoder. + Args: + container (container): PyAV container. + start_pts (int): the starting Presentation TimeStamp to fetch the + video frames. + end_pts (int): the ending Presentation TimeStamp of the decoded frames. + stream (stream): PyAV stream. + stream_name (dict): a dictionary of streams. For example, {"video": 0} + means video stream at stream index 0. + buffer_size (int): number of additional frames to decode beyond end_pts. + Returns: + result (list): list of frames decoded. + max_pts (int): max Presentation TimeStamp of the video sequence. + """ + # Seeking in the stream is imprecise. Thus, seek to an ealier PTS by a + # margin pts. + margin = 1024 + seek_offset = max(start_pts - margin, 0) + + container.seek(seek_offset, any_frame=False, backward=True, stream=stream) + frames = {} + buffer_count = 0 + max_pts = 0 + for frame in container.decode(**stream_name): + max_pts = max(max_pts, frame.pts) + if frame.pts < start_pts: + continue + if frame.pts <= end_pts: + frames[frame.pts] = frame + else: + buffer_count += 1 + frames[frame.pts] = frame + if buffer_count >= buffer_size: + break + result = [frames[pts] for pts in sorted(frames)] + return result, max_pts + + +def torchvision_decode( + video_handle, + sampling_rate, + num_frames, + clip_idx, + video_meta, + num_clips=10, + target_fps=30, + modalities=("visual",), + max_spatial_scale=0, + use_offset=False, +): + """ + If video_meta is not empty, perform temporal selective decoding to sample a + clip from the video with TorchVision decoder. If video_meta is empty, decode + the entire video and update the video_meta. + Args: + video_handle (bytes): raw bytes of the video file. + sampling_rate (int): frame sampling rate (interval between two sampled + frames). + num_frames (int): number of frames to sample. + clip_idx (int): if clip_idx is -1, perform random temporal + sampling. If clip_idx is larger than -1, uniformly split the + video to num_clips clips, and select the clip_idx-th video clip. + video_meta (dict): a dict contains VideoMetaData. Details can be found + at `pytorch/vision/torchvision/io/_video_opt.py`. + num_clips (int): overall number of clips to uniformly sample from the + given video. + target_fps (int): the input video may has different fps, convert it to + the target video fps. + modalities (tuple): tuple of modalities to decode. Currently only + support `visual`, planning to support `acoustic` soon. + max_spatial_scale (int): the maximal resolution of the spatial shorter + edge size during decoding. + Returns: + frames (tensor): decoded frames from the video. + fps (float): the number of frames per second of the video. + decode_all_video (bool): if True, the entire video was decoded. + """ + # Convert the bytes to a tensor. + video_tensor = torch.from_numpy(np.frombuffer(video_handle, dtype=np.uint8)) + + decode_all_video = True + video_start_pts, video_end_pts = 0, -1 + # The video_meta is empty, fetch the meta data from the raw video. + if len(video_meta) == 0: + # Tracking the meta info for selective decoding in the future. + meta = io._probe_video_from_memory(video_tensor) + # Using the information from video_meta to perform selective decoding. + video_meta["video_timebase"] = meta.video_timebase + video_meta["video_numerator"] = meta.video_timebase.numerator + video_meta["video_denominator"] = meta.video_timebase.denominator + video_meta["has_video"] = meta.has_video + video_meta["video_duration"] = meta.video_duration + video_meta["video_fps"] = meta.video_fps + video_meta["audio_timebas"] = meta.audio_timebase + video_meta["audio_numerator"] = meta.audio_timebase.numerator + video_meta["audio_denominator"] = meta.audio_timebase.denominator + video_meta["has_audio"] = meta.has_audio + video_meta["audio_duration"] = meta.audio_duration + video_meta["audio_sample_rate"] = meta.audio_sample_rate + + fps = video_meta["video_fps"] + if ( + video_meta["has_video"] + and video_meta["video_denominator"] > 0 + and video_meta["video_duration"] > 0 + ): + # try selective decoding. + decode_all_video = False + clip_size = sampling_rate * num_frames / target_fps * fps + start_idx, end_idx = get_start_end_idx( + fps * video_meta["video_duration"], + clip_size, + clip_idx, + num_clips, + use_offset=use_offset, + ) + # Convert frame index to pts. + pts_per_frame = video_meta["video_denominator"] / fps + video_start_pts = int(start_idx * pts_per_frame) + video_end_pts = int(end_idx * pts_per_frame) + + # Decode the raw video with the tv decoder. + v_frames, _ = io._read_video_from_memory( + video_tensor, + seek_frame_margin=1.0, + read_video_stream="visual" in modalities, + video_width=0, + video_height=0, + video_min_dimension=max_spatial_scale, + video_pts_range=(video_start_pts, video_end_pts), + video_timebase_numerator=video_meta["video_numerator"], + video_timebase_denominator=video_meta["video_denominator"], + ) + + if v_frames.shape == torch.Size([0]): + # failed selective decoding + decode_all_video = True + video_start_pts, video_end_pts = 0, -1 + v_frames, _ = io._read_video_from_memory( + video_tensor, + seek_frame_margin=1.0, + read_video_stream="visual" in modalities, + video_width=0, + video_height=0, + video_min_dimension=max_spatial_scale, + video_pts_range=(video_start_pts, video_end_pts), + video_timebase_numerator=video_meta["video_numerator"], + video_timebase_denominator=video_meta["video_denominator"], + ) + + return v_frames, fps, decode_all_video + + +def pyav_decode( + container, + sampling_rate, + num_frames, + clip_idx, + num_clips=10, + target_fps=30, + use_offset=False, +): + """ + Convert the video from its original fps to the target_fps. If the video + support selective decoding (contain decoding information in the video head), + the perform temporal selective decoding and sample a clip from the video + with the PyAV decoder. If the video does not support selective decoding, + decode the entire video. + + Args: + container (container): pyav container. + sampling_rate (int): frame sampling rate (interval between two sampled + frames. + num_frames (int): number of frames to sample. + clip_idx (int): if clip_idx is -1, perform random temporal sampling. If + clip_idx is larger than -1, uniformly split the video to num_clips + clips, and select the clip_idx-th video clip. + num_clips (int): overall number of clips to uniformly sample from the + given video. + target_fps (int): the input video may has different fps, convert it to + the target video fps before frame sampling. + Returns: + frames (tensor): decoded frames from the video. Return None if the no + video stream was found. + fps (float): the number of frames per second of the video. + decode_all_video (bool): If True, the entire video was decoded. + """ + # Try to fetch the decoding information from the video head. Some of the + # videos does not support fetching the decoding information, for that case + # it will get None duration. + fps = float(container.streams.video[0].average_rate) + frames_length = container.streams.video[0].frames + duration = container.streams.video[0].duration + + if duration is None: + # If failed to fetch the decoding information, decode the entire video. + decode_all_video = True + video_start_pts, video_end_pts = 0, math.inf + else: + # Perform selective decoding. + decode_all_video = False + start_idx, end_idx = get_start_end_idx( + frames_length, + sampling_rate * num_frames / target_fps * fps, + clip_idx, + num_clips, + use_offset=use_offset, + ) + timebase = duration / frames_length + video_start_pts = int(start_idx * timebase) + video_end_pts = int(end_idx * timebase) + + frames = None + # If video stream was found, fetch video frames from the video. + if container.streams.video: + video_frames, max_pts = pyav_decode_stream( + container, + video_start_pts, + video_end_pts, + container.streams.video[0], + {"video": 0}, + ) + container.close() + + frames = [frame.to_rgb().to_ndarray() for frame in video_frames] + frames = torch.as_tensor(np.stack(frames)) + return frames, fps, decode_all_video + + +def decode( + container, + sampling_rate, + num_frames, + clip_idx=-1, + num_clips=10, + video_meta=None, + target_fps=30, + backend="pyav", + max_spatial_scale=0, + use_offset=False, +): + """ + Decode the video and perform temporal sampling. + Args: + container (container): pyav container. + sampling_rate (int): frame sampling rate (interval between two sampled + frames). + num_frames (int): number of frames to sample. + clip_idx (int): if clip_idx is -1, perform random temporal + sampling. If clip_idx is larger than -1, uniformly split the + video to num_clips clips, and select the + clip_idx-th video clip. + num_clips (int): overall number of clips to uniformly + sample from the given video. + video_meta (dict): a dict contains VideoMetaData. Details can be find + at `pytorch/vision/torchvision/io/_video_opt.py`. + target_fps (int): the input video may have different fps, convert it to + the target video fps before frame sampling. + backend (str): decoding backend includes `pyav` and `torchvision`. The + default one is `pyav`. + max_spatial_scale (int): keep the aspect ratio and resize the frame so + that shorter edge size is max_spatial_scale. Only used in + `torchvision` backend. + Returns: + frames (tensor): decoded frames from the video. + """ + # Currently support two decoders: 1) PyAV, and 2) TorchVision. + assert clip_idx >= -1, "Not valied clip_idx {}".format(clip_idx) + try: + if backend == "pyav": + frames, fps, decode_all_video = pyav_decode( + container, + sampling_rate, + num_frames, + clip_idx, + num_clips, + target_fps, + use_offset=use_offset, + ) + elif backend == "torchvision": + frames, fps, decode_all_video = torchvision_decode( + container, + sampling_rate, + num_frames, + clip_idx, + video_meta, + num_clips, + target_fps, + ("visual",), + max_spatial_scale, + use_offset=use_offset, + ) + else: + raise NotImplementedError( + "Unknown decoding backend {}".format(backend) + ) + except Exception as e: + print("Failed to decode by {} with exception: {}".format(backend, e)) + return None + + # Return None if the frames was not decoded successfully. + if frames is None or frames.size(0) == 0: + return None + + clip_sz = sampling_rate * num_frames / target_fps * fps + start_idx, end_idx = get_start_end_idx( + frames.shape[0], + clip_sz, + clip_idx if decode_all_video else 0, + num_clips if decode_all_video else 1, + use_offset=use_offset, + ) + # Perform temporal sampling from the decoded video. + frames = temporal_sampling(frames, start_idx, end_idx, num_frames) + return frames diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/imagenet.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/imagenet.py" new file mode 100644 index 0000000..6003f4e --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/imagenet.py" @@ -0,0 +1,203 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import json +import numpy as np +import os +import random +import re +import torch +import torch.utils.data + +# import cv2 +from iopath.common.file_io import g_pathmgr +from PIL import Image +from torchvision import transforms as transforms_tv + +import slowfast.datasets.transform as transform +import slowfast.utils.logging as logging + +from .build import DATASET_REGISTRY +from .transform import transforms_imagenet_train + +logger = logging.get_logger(__name__) + + +@DATASET_REGISTRY.register() +class Imagenet(torch.utils.data.Dataset): + """ImageNet dataset.""" + + def __init__(self, cfg, mode, num_retries=10): + self.num_retries = num_retries + self.cfg = cfg + self.mode = mode + self.data_path = cfg.DATA.PATH_TO_DATA_DIR + assert mode in [ + "train", + "val", + "test", + ], "Split '{}' not supported for ImageNet".format(mode) + logger.info("Constructing ImageNet {}...".format(mode)) + if cfg.DATA.PATH_TO_PRELOAD_IMDB == "": + self._construct_imdb() + else: + self._load_imdb() + + def _load_imdb(self): + split_path = os.path.join( + self.cfg.DATA.PATH_TO_PRELOAD_IMDB, f"{self.mode}.json" + ) + with g_pathmgr.open(split_path, "r") as f: + data = f.read() + self._imdb = json.loads(data) + + def _construct_imdb(self): + """Constructs the imdb.""" + # Compile the split data path + split_path = os.path.join(self.data_path, self.mode) + logger.info("{} data path: {}".format(self.mode, split_path)) + # Images are stored per class in subdirs (format: n) + split_files = g_pathmgr.ls(split_path) + self._class_ids = sorted( + f for f in split_files if re.match(r"^n[0-9]+$", f) + ) + # Map ImageNet class ids to contiguous ids + self._class_id_cont_id = {v: i for i, v in enumerate(self._class_ids)} + # Construct the image db + self._imdb = [] + for class_id in self._class_ids: + cont_id = self._class_id_cont_id[class_id] + im_dir = os.path.join(split_path, class_id) + for im_name in g_pathmgr.ls(im_dir): + im_path = os.path.join(im_dir, im_name) + self._imdb.append({"im_path": im_path, "class": cont_id}) + logger.info("Number of images: {}".format(len(self._imdb))) + logger.info("Number of classes: {}".format(len(self._class_ids))) + + def load_image(self, im_path): + """Prepares the image for network input with format of CHW RGB float""" + with g_pathmgr.open(im_path, "rb") as f: + with Image.open(f) as im: + im = im.convert("RGB") + im = torch.from_numpy(np.array(im).astype(np.float32) / 255.0) + # H W C to C H W + im = im.permute([2, 0, 1]) + return im + + def _prepare_im_res(self, im_path): + # Prepare resnet style augmentation. + im = self.load_image(im_path) + # Train and test setups differ + train_size, test_size = ( + self.cfg.DATA.TRAIN_CROP_SIZE, + self.cfg.DATA.TEST_CROP_SIZE, + ) + if self.mode == "train": + # For training use random_sized_crop, horizontal_flip, augment, lighting + im = transform.random_sized_crop_img( + im, + train_size, + jitter_scale=self.cfg.DATA.TRAIN_JITTER_SCALES_RELATIVE, + jitter_aspect=self.cfg.DATA.TRAIN_JITTER_ASPECT_RELATIVE, + ) + im, _ = transform.horizontal_flip(prob=0.5, images=im) + # im = transforms.augment(im, cfg.TRAIN.AUGMENT) + im = transform.lighting_jitter( + im, + 0.1, + self.cfg.DATA.TRAIN_PCA_EIGVAL, + self.cfg.DATA.TRAIN_PCA_EIGVEC, + ) + else: + # For testing use scale and center crop + im, _ = transform.uniform_crop( + im, test_size, spatial_idx=1, scale_size=train_size + ) + # For training and testing use color normalization + im = transform.color_normalization( + im, self.cfg.DATA.MEAN, self.cfg.DATA.STD + ) + # Convert HWC/RGB/float to CHW/BGR/float format + # im = np.ascontiguousarray(im[:, :, ::-1].transpose([2, 0, 1])) + return im + + def _prepare_im_tf(self, im_path): + with g_pathmgr.open(im_path, "rb") as f: + with Image.open(f) as im: + im = im.convert("RGB") + # Convert HWC/BGR/int to HWC/RGB/float format for applying transforms + train_size, test_size = ( + self.cfg.DATA.TRAIN_CROP_SIZE, + self.cfg.DATA.TEST_CROP_SIZE, + ) + + if self.mode == "train": + aug_transform = transforms_imagenet_train( + img_size=(train_size, train_size), + color_jitter=self.cfg.AUG.COLOR_JITTER, + auto_augment=self.cfg.AUG.AA_TYPE, + interpolation=self.cfg.AUG.INTERPOLATION, + re_prob=self.cfg.AUG.RE_PROB, + re_mode=self.cfg.AUG.RE_MODE, + re_count=self.cfg.AUG.RE_COUNT, + mean=self.cfg.DATA.MEAN, + std=self.cfg.DATA.STD, + ) + else: + t = [] + size = int((256 / 224) * test_size) + t.append( + transforms_tv.Resize( + size, interpolation=3 + ), # to maintain same ratio w.r.t. 224 images + ) + t.append(transforms_tv.CenterCrop(test_size)) + t.append(transforms_tv.ToTensor()) + t.append( + transforms_tv.Normalize(self.cfg.DATA.MEAN, self.cfg.DATA.STD) + ) + aug_transform = transforms_tv.Compose(t) + im = aug_transform(im) + return im + + def __load__(self, index): + try: + # Load the image + im_path = self._imdb[index]["im_path"] + # Prepare the image for training / testing + if self.cfg.AUG.ENABLE: + if self.mode == "train" and self.cfg.AUG.NUM_SAMPLE > 1: + im = [] + for _ in range(self.cfg.AUG.NUM_SAMPLE): + crop = self._prepare_im_tf(im_path) + im.append(crop) + return im + else: + im = self._prepare_im_tf(im_path) + return im + else: + im = self._prepare_im_res(im_path) + return im + except Exception: + return None + + def __getitem__(self, index): + # if the current image is corrupted, load a different image. + for _ in range(self.num_retries): + im = self.__load__(index) + # Data corrupted, retry with a different image. + if im is None: + index = random.randint(0, len(self._imdb) - 1) + else: + break + # Retrieve the label + label = self._imdb[index]["class"] + if isinstance(im, list): + label = [label for _ in range(len(im))] + dummy = [torch.Tensor() for _ in range(len(im))] + return im, label, dummy, {} + else: + dummy = torch.Tensor() + return [im], label, dummy, {} + + def __len__(self): + return len(self._imdb) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/kinetics.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/kinetics.py" new file mode 100644 index 0000000..822952d --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/kinetics.py" @@ -0,0 +1,408 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import os +import random +import torch +import torch.utils.data +from iopath.common.file_io import g_pathmgr +from torchvision import transforms + +import slowfast.utils.logging as logging + +from . import decoder as decoder +from . import utils as utils +from . import video_container as container +from .build import DATASET_REGISTRY +from .random_erasing import RandomErasing +from .transform import create_random_augment + +logger = logging.get_logger(__name__) + + +@DATASET_REGISTRY.register() +class Kinetics(torch.utils.data.Dataset): + """ + Kinetics video loader. Construct the Kinetics video loader, then sample + clips from the videos. For training and validation, a single clip is + randomly sampled from every video with random cropping, scaling, and + flipping. For testing, multiple clips are uniformaly sampled from every + video with uniform cropping. For uniform cropping, we take the left, center, + and right crop if the width is larger than height, or take top, center, and + bottom crop if the height is larger than the width. + """ + + def __init__(self, cfg, mode, num_retries=10): + """ + Construct the Kinetics video loader with a given csv file. The format of + the csv file is: + ``` + path_to_video_1 label_1 + path_to_video_2 label_2 + ... + path_to_video_N label_N + ``` + Args: + cfg (CfgNode): configs. + mode (string): Options includes `train`, `val`, or `test` mode. + For the train and val mode, the data loader will take data + from the train or val set, and sample one clip per video. + For the test mode, the data loader will take data from test set, + and sample multiple clips per video. + num_retries (int): number of retries. + """ + # Only support train, val, and test mode. + assert mode in [ + "train", + "val", + "test", + ], "Split '{}' not supported for Kinetics".format(mode) + self.mode = mode + self.cfg = cfg + + self._video_meta = {} + self._num_retries = num_retries + # For training or validation mode, one single clip is sampled from every + # video. For testing, NUM_ENSEMBLE_VIEWS clips are sampled from every + # video. For every clip, NUM_SPATIAL_CROPS is cropped spatially from + # the frames. + if self.mode in ["train", "val"]: + self._num_clips = 1 + elif self.mode in ["test"]: + self._num_clips = ( + cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS + ) + + logger.info("Constructing Kinetics {}...".format(mode)) + self._construct_loader() + self.aug = False + self.rand_erase = False + self.use_temporal_gradient = False + self.temporal_gradient_rate = 0.0 + + if self.mode == "train" and self.cfg.AUG.ENABLE: + self.aug = True + if self.cfg.AUG.RE_PROB > 0: + self.rand_erase = True + + def _construct_loader(self): + """ + Construct the video loader. + """ + path_to_file = os.path.join( + self.cfg.DATA.PATH_TO_DATA_DIR, "{}.csv".format(self.mode) + ) + assert g_pathmgr.exists(path_to_file), "{} dir not found".format( + path_to_file + ) + + self._path_to_videos = [] + self._labels = [] + self._spatial_temporal_idx = [] + with g_pathmgr.open(path_to_file, "r") as f: + for clip_idx, path_label in enumerate(f.read().splitlines()): + assert ( + len(path_label.split(self.cfg.DATA.PATH_LABEL_SEPARATOR)) + == 2 + ) + path, label = path_label.split( + self.cfg.DATA.PATH_LABEL_SEPARATOR + ) + for idx in range(self._num_clips): + self._path_to_videos.append( + os.path.join(self.cfg.DATA.PATH_PREFIX, path) + ) + self._labels.append(int(label)) + self._spatial_temporal_idx.append(idx) + self._video_meta[clip_idx * self._num_clips + idx] = {} + assert ( + len(self._path_to_videos) > 0 + ), "Failed to load Kinetics split {} from {}".format( + self._split_idx, path_to_file + ) + logger.info( + "Constructing kinetics dataloader (size: {}) from {}".format( + len(self._path_to_videos), path_to_file + ) + ) + + def __getitem__(self, index): + """ + Given the video index, return the list of frames, label, and video + index if the video can be fetched and decoded successfully, otherwise + repeatly find a random video that can be decoded as a replacement. + Args: + index (int): the video index provided by the pytorch sampler. + Returns: + frames (tensor): the frames of sampled from the video. The dimension + is `channel` x `num frames` x `height` x `width`. + label (int): the label of the current video. + index (int): if the video provided by pytorch sampler can be + decoded, then return the index of the video. If not, return the + index of the video replacement that can be decoded. + """ + short_cycle_idx = None + # When short cycle is used, input index is a tupple. + if isinstance(index, tuple): + index, short_cycle_idx = index + + if self.mode in ["train", "val"]: + # -1 indicates random sampling. + temporal_sample_index = -1 + spatial_sample_index = -1 + min_scale = self.cfg.DATA.TRAIN_JITTER_SCALES[0] + max_scale = self.cfg.DATA.TRAIN_JITTER_SCALES[1] + crop_size = self.cfg.DATA.TRAIN_CROP_SIZE + if short_cycle_idx in [0, 1]: + crop_size = int( + round( + self.cfg.MULTIGRID.SHORT_CYCLE_FACTORS[short_cycle_idx] + * self.cfg.MULTIGRID.DEFAULT_S + ) + ) + if self.cfg.MULTIGRID.DEFAULT_S > 0: + # Decreasing the scale is equivalent to using a larger "span" + # in a sampling grid. + min_scale = int( + round( + float(min_scale) + * crop_size + / self.cfg.MULTIGRID.DEFAULT_S + ) + ) + elif self.mode in ["test"]: + temporal_sample_index = ( + self._spatial_temporal_idx[index] + // self.cfg.TEST.NUM_SPATIAL_CROPS + ) + # spatial_sample_index is in [0, 1, 2]. Corresponding to left, + # center, or right if width is larger than height, and top, middle, + # or bottom if height is larger than width. + spatial_sample_index = ( + ( + self._spatial_temporal_idx[index] + % self.cfg.TEST.NUM_SPATIAL_CROPS + ) + if self.cfg.TEST.NUM_SPATIAL_CROPS > 1 + else 1 + ) + min_scale, max_scale, crop_size = ( + [self.cfg.DATA.TEST_CROP_SIZE] * 3 + if self.cfg.TEST.NUM_SPATIAL_CROPS > 1 + else [self.cfg.DATA.TRAIN_JITTER_SCALES[0]] * 2 + + [self.cfg.DATA.TEST_CROP_SIZE] + ) + # The testing is deterministic and no jitter should be performed. + # min_scale, max_scale, and crop_size are expect to be the same. + assert len({min_scale, max_scale}) == 1 + else: + raise NotImplementedError( + "Does not support {} mode".format(self.mode) + ) + sampling_rate = utils.get_random_sampling_rate( + self.cfg.MULTIGRID.LONG_CYCLE_SAMPLING_RATE, + self.cfg.DATA.SAMPLING_RATE, + ) + # Try to decode and sample a clip from a video. If the video can not be + # decoded, repeatly find a random video replacement that can be decoded. + for i_try in range(self._num_retries): + video_container = None + try: + video_container = container.get_video_container( + self._path_to_videos[index], + self.cfg.DATA_LOADER.ENABLE_MULTI_THREAD_DECODE, + self.cfg.DATA.DECODING_BACKEND, + ) + except Exception as e: + logger.info( + "Failed to load video from {} with error {}".format( + self._path_to_videos[index], e + ) + ) + # Select a random video if the current video was not able to access. + if video_container is None: + logger.warning( + "Failed to meta load video idx {} from {}; trial {}".format( + index, self._path_to_videos[index], i_try + ) + ) + if self.mode not in ["test"] and i_try > self._num_retries // 2: + # let's try another one + index = random.randint(0, len(self._path_to_videos) - 1) + continue + + # Decode video. Meta info is used to perform selective decoding. + frames = decoder.decode( + video_container, + sampling_rate, + self.cfg.DATA.NUM_FRAMES, + temporal_sample_index, + self.cfg.TEST.NUM_ENSEMBLE_VIEWS, + video_meta=self._video_meta[index], + target_fps=self.cfg.DATA.TARGET_FPS, + backend=self.cfg.DATA.DECODING_BACKEND, + max_spatial_scale=min_scale, + use_offset=self.cfg.DATA.USE_OFFSET_SAMPLING, + ) + + # If decoding failed (wrong format, video is too short, and etc), + # select another video. + if frames is None: + logger.warning( + "Failed to decode video idx {} from {}; trial {}".format( + index, self._path_to_videos[index], i_try + ) + ) + if self.mode not in ["test"] and i_try > self._num_retries // 2: + # let's try another one + index = random.randint(0, len(self._path_to_videos) - 1) + continue + + if self.aug: + if self.cfg.AUG.NUM_SAMPLE > 1: + + frame_list = [] + label_list = [] + index_list = [] + for _ in range(self.cfg.AUG.NUM_SAMPLE): + new_frames = self._aug_frame( + frames, + spatial_sample_index, + min_scale, + max_scale, + crop_size, + ) + label = self._labels[index] + new_frames = utils.pack_pathway_output( + self.cfg, new_frames + ) + frame_list.append(new_frames) + label_list.append(label) + index_list.append(index) + return frame_list, label_list, index_list, {} + + else: + frames = self._aug_frame( + frames, + spatial_sample_index, + min_scale, + max_scale, + crop_size, + ) + + else: + frames = utils.tensor_normalize( + frames, self.cfg.DATA.MEAN, self.cfg.DATA.STD + ) + # T H W C -> C T H W. + frames = frames.permute(3, 0, 1, 2) + # Perform data augmentation. + frames = utils.spatial_sampling( + frames, + spatial_idx=spatial_sample_index, + min_scale=min_scale, + max_scale=max_scale, + crop_size=crop_size, + random_horizontal_flip=self.cfg.DATA.RANDOM_FLIP, + inverse_uniform_sampling=self.cfg.DATA.INV_UNIFORM_SAMPLE, + ) + + label = self._labels[index] + frames = utils.pack_pathway_output(self.cfg, frames) + return frames, label, index, {} + else: + raise RuntimeError( + "Failed to fetch video after {} retries.".format( + self._num_retries + ) + ) + + def _aug_frame( + self, + frames, + spatial_sample_index, + min_scale, + max_scale, + crop_size, + ): + aug_transform = create_random_augment( + input_size=(frames.size(1), frames.size(2)), + auto_augment=self.cfg.AUG.AA_TYPE, + interpolation=self.cfg.AUG.INTERPOLATION, + ) + # T H W C -> T C H W. + frames = frames.permute(0, 3, 1, 2) + list_img = self._frame_to_list_img(frames) + list_img = aug_transform(list_img) + frames = self._list_img_to_frames(list_img) + frames = frames.permute(0, 2, 3, 1) + + frames = utils.tensor_normalize( + frames, self.cfg.DATA.MEAN, self.cfg.DATA.STD + ) + # T H W C -> C T H W. + frames = frames.permute(3, 0, 1, 2) + # Perform data augmentation. + scl, asp = ( + self.cfg.DATA.TRAIN_JITTER_SCALES_RELATIVE, + self.cfg.DATA.TRAIN_JITTER_ASPECT_RELATIVE, + ) + relative_scales = ( + None if (self.mode not in ["train"] or len(scl) == 0) else scl + ) + relative_aspect = ( + None if (self.mode not in ["train"] or len(asp) == 0) else asp + ) + frames = utils.spatial_sampling( + frames, + spatial_idx=spatial_sample_index, + min_scale=min_scale, + max_scale=max_scale, + crop_size=crop_size, + random_horizontal_flip=self.cfg.DATA.RANDOM_FLIP, + inverse_uniform_sampling=self.cfg.DATA.INV_UNIFORM_SAMPLE, + aspect_ratio=relative_aspect, + scale=relative_scales, + motion_shift=self.cfg.DATA.TRAIN_JITTER_MOTION_SHIFT + if self.mode in ["train"] + else False, + ) + + if self.rand_erase: + erase_transform = RandomErasing( + self.cfg.AUG.RE_PROB, + mode=self.cfg.AUG.RE_MODE, + max_count=self.cfg.AUG.RE_COUNT, + num_splits=self.cfg.AUG.RE_COUNT, + device="cpu", + ) + frames = frames.permute(1, 0, 2, 3) + frames = erase_transform(frames) + frames = frames.permute(1, 0, 2, 3) + + return frames + + def _frame_to_list_img(self, frames): + img_list = [ + transforms.ToPILImage()(frames[i]) for i in range(frames.size(0)) + ] + return img_list + + def _list_img_to_frames(self, img_list): + img_list = [transforms.ToTensor()(img) for img in img_list] + return torch.stack(img_list) + + def __len__(self): + """ + Returns: + (int): the number of videos in the dataset. + """ + return self.num_videos + + @property + def num_videos(self): + """ + Returns: + (int): the number of videos in the dataset. + """ + return len(self._path_to_videos) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/loader.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/loader.py" new file mode 100644 index 0000000..fb96cb1 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/loader.py" @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Data loader.""" + +import itertools +import numpy as np +from functools import partial +import torch +from torch.utils.data._utils.collate import default_collate +from torch.utils.data.distributed import DistributedSampler +from torch.utils.data.sampler import RandomSampler + +from slowfast.datasets.multigrid_helper import ShortCycleBatchSampler + +from . import utils as utils +from .build import build_dataset + + +def multiple_samples_collate(batch, fold=False): + """ + Collate function for repeated augmentation. Each instance in the batch has + more than one sample. + Args: + batch (tuple or list): data batch to collate. + Returns: + (tuple): collated data batch. + """ + inputs, labels, video_idx, extra_data = zip(*batch) + inputs = [item for sublist in inputs for item in sublist] + labels = [item for sublist in labels for item in sublist] + video_idx = [item for sublist in video_idx for item in sublist] + inputs, labels, video_idx, extra_data = ( + default_collate(inputs), + default_collate(labels), + default_collate(video_idx), + default_collate(extra_data), + ) + if fold: + return [inputs], labels, video_idx, extra_data + else: + return inputs, labels, video_idx, extra_data + + +def detection_collate(batch): + """ + Collate function for detection task. Concatanate bboxes, labels and + metadata from different samples in the first dimension instead of + stacking them to have a batch-size dimension. + Args: + batch (tuple or list): data batch to collate. + Returns: + (tuple): collated detection data batch. + """ + inputs, labels, video_idx, extra_data = zip(*batch) + inputs, video_idx = default_collate(inputs), default_collate(video_idx) + labels = torch.tensor(np.concatenate(labels, axis=0)).float() + + collated_extra_data = {} + for key in extra_data[0].keys(): + data = [d[key] for d in extra_data] + if key == "boxes" or key == "ori_boxes": + # Append idx info to the bboxes before concatenating them. + bboxes = [ + np.concatenate( + [np.full((data[i].shape[0], 1), float(i)), data[i]], axis=1 + ) + for i in range(len(data)) + ] + bboxes = np.concatenate(bboxes, axis=0) + collated_extra_data[key] = torch.tensor(bboxes).float() + elif key == "metadata": + collated_extra_data[key] = torch.tensor( + list(itertools.chain(*data)) + ).view(-1, 2) + else: + collated_extra_data[key] = default_collate(data) + + return inputs, labels, video_idx, collated_extra_data + + +def construct_loader(cfg, split, is_precise_bn=False): + """ + Constructs the data loader for the given dataset. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + split (str): the split of the data loader. Options include `train`, + `val`, and `test`. + """ + assert split in ["train", "val", "test"] + if split in ["train"]: + dataset_name = cfg.TRAIN.DATASET + batch_size = int(cfg.TRAIN.BATCH_SIZE / max(1, cfg.NUM_GPUS)) + shuffle = True + drop_last = True + elif split in ["val"]: + dataset_name = cfg.TRAIN.DATASET + batch_size = int(cfg.TRAIN.BATCH_SIZE / max(1, cfg.NUM_GPUS)) + shuffle = False + drop_last = False + elif split in ["test"]: + dataset_name = cfg.TEST.DATASET + batch_size = int(cfg.TEST.BATCH_SIZE / max(1, cfg.NUM_GPUS)) + shuffle = False + drop_last = False + + # Construct the dataset + dataset = build_dataset(dataset_name, cfg, split) + + if isinstance(dataset, torch.utils.data.IterableDataset): + loader = torch.utils.data.DataLoader( + dataset, + batch_size=batch_size, + num_workers=cfg.DATA_LOADER.NUM_WORKERS, + pin_memory=cfg.DATA_LOADER.PIN_MEMORY, + drop_last=drop_last, + collate_fn=detection_collate if cfg.DETECTION.ENABLE else None, + worker_init_fn=utils.loader_worker_init_fn(dataset), + ) + else: + if ( + cfg.MULTIGRID.SHORT_CYCLE + and split in ["train"] + and not is_precise_bn + ): + # Create a sampler for multi-process training + sampler = utils.create_sampler(dataset, shuffle, cfg) + batch_sampler = ShortCycleBatchSampler( + sampler, batch_size=batch_size, drop_last=drop_last, cfg=cfg + ) + # Create a loader + loader = torch.utils.data.DataLoader( + dataset, + batch_sampler=batch_sampler, + num_workers=cfg.DATA_LOADER.NUM_WORKERS, + pin_memory=cfg.DATA_LOADER.PIN_MEMORY, + worker_init_fn=utils.loader_worker_init_fn(dataset), + ) + else: + # Create a sampler for multi-process training + sampler = utils.create_sampler(dataset, shuffle, cfg) + # Create a loader + if cfg.DETECTION.ENABLE: + collate_func = detection_collate + elif cfg.AUG.NUM_SAMPLE > 1 and split in ["train"]: + collate_func = partial( + multiple_samples_collate, fold="imagenet" in dataset_name + ) + else: + collate_func = None + + loader = torch.utils.data.DataLoader( + dataset, + batch_size=batch_size, + shuffle=(False if sampler else shuffle), + sampler=sampler, + num_workers=cfg.DATA_LOADER.NUM_WORKERS, + pin_memory=cfg.DATA_LOADER.PIN_MEMORY, + drop_last=drop_last, + collate_fn=collate_func, + worker_init_fn=utils.loader_worker_init_fn(dataset), + ) + return loader + + +def shuffle_dataset(loader, cur_epoch): + """ " + Shuffles the data. + Args: + loader (loader): data loader to perform shuffle. + cur_epoch (int): number of the current epoch. + """ + if ( + loader._dataset_kind + == torch.utils.data.dataloader._DatasetKind.Iterable + ): + if hasattr(loader.dataset, "sampler"): + sampler = loader.dataset.sampler + else: + raise RuntimeError( + "Unknown sampler for IterableDataset when shuffling dataset" + ) + else: + sampler = ( + loader.batch_sampler.sampler + if isinstance(loader.batch_sampler, ShortCycleBatchSampler) + else loader.sampler + ) + assert isinstance( + sampler, (RandomSampler, DistributedSampler) + ), "Sampler type '{}' not supported".format(type(sampler)) + # RandomSampler handles shuffling automatically + if isinstance(sampler, DistributedSampler): + # DistributedSampler shuffles data based on epoch + sampler.set_epoch(cur_epoch) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/mixup.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/mixup.py" new file mode 100644 index 0000000..a75846d --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/mixup.py" @@ -0,0 +1,189 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +""" +This implementation is based on +https://github.com/rwightman/pytorch-image-models/blob/master/timm/data/mixup.py, +published under an Apache License 2.0. + +COMMENT FROM ORIGINAL: +Mixup and Cutmix +Papers: +mixup: Beyond Empirical Risk Minimization (https://arxiv.org/abs/1710.09412) +CutMix: Regularization Strategy to Train Strong Classifiers with Localizable Features (https://arxiv.org/abs/1905.04899) # NOQA +Code Reference: +CutMix: https://github.com/clovaai/CutMix-PyTorch +Hacked together by / Copyright 2020 Ross Wightman +""" + +import numpy as np +import torch + + +def convert_to_one_hot(targets, num_classes, on_value=1.0, off_value=0.0): + """ + This function converts target class indices to one-hot vectors, given the + number of classes. + Args: + targets (loader): Class labels. + num_classes (int): Total number of classes. + on_value (float): Target Value for ground truth class. + off_value (float): Target Value for other classes.This value is used for + label smoothing. + """ + + targets = targets.long().view(-1, 1) + return torch.full( + (targets.size()[0], num_classes), off_value, device=targets.device + ).scatter_(1, targets, on_value) + + +def mixup_target(target, num_classes, lam=1.0, smoothing=0.0): + """ + This function converts target class indices to one-hot vectors, given the + number of classes. + Args: + targets (loader): Class labels. + num_classes (int): Total number of classes. + lam (float): lamba value for mixup/cutmix. + smoothing (float): Label smoothing value. + """ + off_value = smoothing / num_classes + on_value = 1.0 - smoothing + off_value + target1 = convert_to_one_hot( + target, + num_classes, + on_value=on_value, + off_value=off_value, + ) + target2 = convert_to_one_hot( + target.flip(0), + num_classes, + on_value=on_value, + off_value=off_value, + ) + return target1 * lam + target2 * (1.0 - lam) + + +def rand_bbox(img_shape, lam, margin=0.0, count=None): + """ + Generates a random square bbox based on lambda value. + + Args: + img_shape (tuple): Image shape as tuple + lam (float): Cutmix lambda value + margin (float): Percentage of bbox dimension to enforce as margin (reduce amount of box outside image) + count (int): Number of bbox to generate + """ + ratio = np.sqrt(1 - lam) + img_h, img_w = img_shape[-2:] + cut_h, cut_w = int(img_h * ratio), int(img_w * ratio) + margin_y, margin_x = int(margin * cut_h), int(margin * cut_w) + cy = np.random.randint(0 + margin_y, img_h - margin_y, size=count) + cx = np.random.randint(0 + margin_x, img_w - margin_x, size=count) + yl = np.clip(cy - cut_h // 2, 0, img_h) + yh = np.clip(cy + cut_h // 2, 0, img_h) + xl = np.clip(cx - cut_w // 2, 0, img_w) + xh = np.clip(cx + cut_w // 2, 0, img_w) + return yl, yh, xl, xh + + +def get_cutmix_bbox(img_shape, lam, correct_lam=True, count=None): + """ + Generates the box coordinates for cutmix. + + Args: + img_shape (tuple): Image shape as tuple + lam (float): Cutmix lambda value + correct_lam (bool): Apply lambda correction when cutmix bbox clipped by + image borders. + count (int): Number of bbox to generate + """ + + yl, yu, xl, xu = rand_bbox(img_shape, lam, count=count) + if correct_lam: + bbox_area = (yu - yl) * (xu - xl) + lam = 1.0 - bbox_area / float(img_shape[-2] * img_shape[-1]) + return (yl, yu, xl, xu), lam + + +class MixUp: + """ + Apply mixup and/or cutmix for videos at batch level. + mixup: Beyond Empirical Risk Minimization (https://arxiv.org/abs/1710.09412) + CutMix: Regularization Strategy to Train Strong Classifiers with Localizable + Features (https://arxiv.org/abs/1905.04899) + """ + + def __init__( + self, + mixup_alpha=1.0, + cutmix_alpha=0.0, + mix_prob=1.0, + switch_prob=0.5, + correct_lam=True, + label_smoothing=0.1, + num_classes=1000, + ): + """ + Args: + mixup_alpha (float): Mixup alpha value. + cutmix_alpha (float): Cutmix alpha value. + mix_prob (float): Probability of applying mixup or cutmix. + switch_prob (float): Probability of switching to cutmix instead of + mixup when both are active. + correct_lam (bool): Apply lambda correction when cutmix bbox + clipped by image borders. + label_smoothing (float): Apply label smoothing to the mixed target + tensor. If label_smoothing is not used, set it to 0. + num_classes (int): Number of classes for target. + """ + self.mixup_alpha = mixup_alpha + self.cutmix_alpha = cutmix_alpha + self.mix_prob = mix_prob + self.switch_prob = switch_prob + self.label_smoothing = label_smoothing + self.num_classes = num_classes + self.correct_lam = correct_lam + + def _get_mixup_params(self): + lam = 1.0 + use_cutmix = False + if np.random.rand() < self.mix_prob: + if self.mixup_alpha > 0.0 and self.cutmix_alpha > 0.0: + use_cutmix = np.random.rand() < self.switch_prob + lam_mix = ( + np.random.beta(self.cutmix_alpha, self.cutmix_alpha) + if use_cutmix + else np.random.beta(self.mixup_alpha, self.mixup_alpha) + ) + elif self.mixup_alpha > 0.0: + lam_mix = np.random.beta(self.mixup_alpha, self.mixup_alpha) + elif self.cutmix_alpha > 0.0: + use_cutmix = True + lam_mix = np.random.beta(self.cutmix_alpha, self.cutmix_alpha) + lam = float(lam_mix) + return lam, use_cutmix + + def _mix_batch(self, x): + lam, use_cutmix = self._get_mixup_params() + if lam == 1.0: + return 1.0 + if use_cutmix: + (yl, yh, xl, xh), lam = get_cutmix_bbox( + x.shape, + lam, + correct_lam=self.correct_lam, + ) + x[..., yl:yh, xl:xh] = x.flip(0)[..., yl:yh, xl:xh] + else: + x_flipped = x.flip(0).mul_(1.0 - lam) + x.mul_(lam).add_(x_flipped) + return lam + + def __call__(self, x, target): + assert len(x) > 1, "Batch size should be greater than 1 for mixup." + lam = self._mix_batch(x) + target = mixup_target( + target, self.num_classes, lam, self.label_smoothing + ) + return x, target diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/multigrid_helper.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/multigrid_helper.py" new file mode 100644 index 0000000..2381a4c --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/multigrid_helper.py" @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Helper functions for multigrid training.""" + +import numpy as np +import torch +from torch.utils.data.sampler import Sampler + +TORCH_MAJOR = int(torch.__version__.split(".")[0]) +TORCH_MINOR = int(torch.__version__.split(".")[1]) + +if TORCH_MAJOR >= 1 and TORCH_MINOR >= 8: + _int_classes = int +else: + from torch._six import int_classes as _int_classes + + +class ShortCycleBatchSampler(Sampler): + """ + Extend Sampler to support "short cycle" sampling. + See paper "A Multigrid Method for Efficiently Training Video Models", + Wu et al., 2019 (https://arxiv.org/abs/1912.00998) for details. + """ + + def __init__(self, sampler, batch_size, drop_last, cfg): + if not isinstance(sampler, Sampler): + raise ValueError( + "sampler should be an instance of " + "torch.utils.data.Sampler, but got sampler={}".format(sampler) + ) + if ( + not isinstance(batch_size, _int_classes) + or isinstance(batch_size, bool) + or batch_size <= 0 + ): + raise ValueError( + "batch_size should be a positive integer value, " + "but got batch_size={}".format(batch_size) + ) + if not isinstance(drop_last, bool): + raise ValueError( + "drop_last should be a boolean value, but got " + "drop_last={}".format(drop_last) + ) + self.sampler = sampler + self.drop_last = drop_last + + bs_factor = [ + int( + round( + ( + float(cfg.DATA.TRAIN_CROP_SIZE) + / (s * cfg.MULTIGRID.DEFAULT_S) + ) + ** 2 + ) + ) + for s in cfg.MULTIGRID.SHORT_CYCLE_FACTORS + ] + + self.batch_sizes = [ + batch_size * bs_factor[0], + batch_size * bs_factor[1], + batch_size, + ] + + def __iter__(self): + counter = 0 + batch_size = self.batch_sizes[0] + batch = [] + for idx in self.sampler: + batch.append((idx, counter % 3)) + if len(batch) == batch_size: + yield batch + counter += 1 + batch_size = self.batch_sizes[counter % 3] + batch = [] + if len(batch) > 0 and not self.drop_last: + yield batch + + def __len__(self): + avg_batch_size = sum(self.batch_sizes) / 3.0 + if self.drop_last: + return int(np.floor(len(self.sampler) / avg_batch_size)) + else: + return int(np.ceil(len(self.sampler) / avg_batch_size)) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ptv_datasets.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ptv_datasets.py" new file mode 100644 index 0000000..61de8c4 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ptv_datasets.py" @@ -0,0 +1,594 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import functools +import os +from typing import Dict +import torch +from torch.utils.data import ( + DistributedSampler, + RandomSampler, + SequentialSampler, +) +from torchvision.transforms import Compose, Lambda +from torchvision.transforms._transforms_video import ( + NormalizeVideo, + RandomCropVideo, + RandomHorizontalFlipVideo, +) + +import slowfast.utils.logging as logging + +from pytorchvideo.data import ( + Charades, + LabeledVideoDataset, + SSv2, + make_clip_sampler, +) +from pytorchvideo.data.labeled_video_paths import LabeledVideoPaths +from pytorchvideo.transforms import ( + ApplyTransformToKey, + RandomShortSideScale, + ShortSideScale, + UniformCropVideo, + UniformTemporalSubsample, +) + +from . import utils as utils +from .build import DATASET_REGISTRY + +logger = logging.get_logger(__name__) + + +class PTVDatasetWrapper(torch.utils.data.IterableDataset): + """ + Wrapper for PyTorchVideo datasets. + """ + + def __init__(self, num_videos, clips_per_video, crops_per_clip, dataset): + """ + Construct the dataset. + + Args: + num_vidoes (int): number of videos in the dataset. + clips_per_video (int): number of clips per video in the dataset. + dataset (torch.utils.data.IterableDataset): a PyTorchVideo dataset. + """ + self._clips_per_video = clips_per_video + self._crops_per_clip = crops_per_clip + self._num_videos = num_videos + self.dataset = dataset + + def __next__(self): + """ + Retrieves the next clip from the dataset. + """ + return self.dataset.__next__() + + @property + def sampler(self): + """ + Returns: + (torch.utils.data.Sampler): video sampler for the dataset. + """ + return self.dataset.video_sampler + + def __len__(self): + """ + Returns: + (int): the number of clips per replica in the IterableDataset. + """ + return len(self.sampler) * self._clips_per_video * self._crops_per_clip + + @property + def num_videos(self): + """ + Returns: + (int): the number of clips in total in the dataset. + """ + return self._num_videos * self._clips_per_video * self._crops_per_clip + + def __iter__(self): + return self + + +class PackPathway(torch.nn.Module): + """ + Transform for converting video frames as a list of tensors. Each tensor + corresponding to a unique pathway. + """ + + def __init__(self, cfg): + super().__init__() + self.cfg = cfg + + def forward(self, x: torch.Tensor): + return utils.pack_pathway_output(self.cfg, x) + + +class DictToTuple(torch.nn.Module): + """ + Transform for converting output from dict to a tuple following PySlowFast + dataset output format. + """ + + def __init__(self, num_clips, num_crops): + super().__init__() + self._num_clips = num_clips + self._num_crops = num_crops + + def forward(self, x: Dict[str, torch.Tensor]): + index = ( + x["video_index"] * self._num_clips * self._num_crops + + x["clip_index"] * self._num_crops + + x["aug_index"] + ) + + return x["video"], x["label"], index, {} + + +def div255(x): + """ + Scale clip frames from [0, 255] to [0, 1]. + Args: + x (Tensor): A tensor of the clip's RGB frames with shape: + (channel, time, height, width). + + Returns: + x (Tensor): Scaled tensor by divide 255. + """ + return x / 255.0 + + +@DATASET_REGISTRY.register() +def Ptvkinetics(cfg, mode): + """ + Construct the Kinetics video loader with a given csv file. The format of + the csv file is: + ``` + path_to_video_1 label_1 + path_to_video_2 label_2 + ... + path_to_video_N label_N + ``` + For `train` and `val` mode, a single clip is randomly sampled from every video + with random cropping, scaling, and flipping. For `test` mode, multiple clips are + uniformaly sampled from every video with center cropping. + Args: + cfg (CfgNode): configs. + mode (string): Options includes `train`, `val`, or `test` mode. + For the train and val mode, the data loader will take data + from the train or val set, and sample one clip per video. + For the test mode, the data loader will take data from test set, + and sample multiple clips per video. + """ + # Only support train, val, and test mode. + assert mode in [ + "train", + "val", + "test", + ], "Split '{}' not supported".format(mode) + + logger.info("Constructing Ptvkinetics {}...".format(mode)) + + clip_duration = ( + cfg.DATA.NUM_FRAMES * cfg.DATA.SAMPLING_RATE / cfg.DATA.TARGET_FPS + ) + path_to_file = os.path.join( + cfg.DATA.PATH_TO_DATA_DIR, "{}.csv".format(mode) + ) + labeled_video_paths = LabeledVideoPaths.from_path(path_to_file) + num_videos = len(labeled_video_paths) + labeled_video_paths.path_prefix = cfg.DATA.PATH_PREFIX + logger.info( + "Constructing kinetics dataloader (size: {}) from {}".format( + num_videos, path_to_file + ) + ) + + if mode in ["train", "val"]: + num_clips = 1 + num_crops = 1 + + transform = Compose( + [ + ApplyTransformToKey( + key="video", + transform=Compose( + [ + UniformTemporalSubsample(cfg.DATA.NUM_FRAMES), + Lambda(div255), + NormalizeVideo(cfg.DATA.MEAN, cfg.DATA.STD), + RandomShortSideScale( + min_size=cfg.DATA.TRAIN_JITTER_SCALES[0], + max_size=cfg.DATA.TRAIN_JITTER_SCALES[1], + ), + RandomCropVideo(cfg.DATA.TRAIN_CROP_SIZE), + ] + + ( + [RandomHorizontalFlipVideo(p=0.5)] + if cfg.DATA.RANDOM_FLIP + else [] + ) + + [PackPathway(cfg)] + ), + ), + DictToTuple(num_clips, num_crops), + ] + ) + + clip_sampler = make_clip_sampler("random", clip_duration) + if cfg.NUM_GPUS > 1: + video_sampler = DistributedSampler + else: + video_sampler = ( + RandomSampler if mode == "train" else SequentialSampler + ) + else: + num_clips = cfg.TEST.NUM_ENSEMBLE_VIEWS + num_crops = cfg.TEST.NUM_SPATIAL_CROPS + + transform = Compose( + [ + ApplyTransformToKey( + key="video", + transform=Compose( + [ + UniformTemporalSubsample(cfg.DATA.NUM_FRAMES), + Lambda(div255), + NormalizeVideo(cfg.DATA.MEAN, cfg.DATA.STD), + ShortSideScale( + size=cfg.DATA.TRAIN_JITTER_SCALES[0] + ), + ] + ), + ), + UniformCropVideo(size=cfg.DATA.TEST_CROP_SIZE), + ApplyTransformToKey(key="video", transform=PackPathway(cfg)), + DictToTuple(num_clips, num_crops), + ] + ) + clip_sampler = make_clip_sampler( + "constant_clips_per_video", + clip_duration, + num_clips, + num_crops, + ) + video_sampler = ( + DistributedSampler if cfg.NUM_GPUS > 1 else SequentialSampler + ) + + return PTVDatasetWrapper( + num_videos=num_videos, + clips_per_video=num_clips, + crops_per_clip=num_crops, + dataset=LabeledVideoDataset( + labeled_video_paths=labeled_video_paths, + clip_sampler=clip_sampler, + video_sampler=video_sampler, + transform=transform, + decode_audio=False, + ), + ) + + +def process_charades_label(x, mode, num_classes): + """ + Process the video label for Charades dataset. Use video-level label for + training mode, otherwise use clip-level label. Then convert the label into + a binary vector. + Args: + x (dict): a video clip including label index. + mode (string): Options includes `train`, `val`, or `test` mode. + num_classes (int): Number of classes in the dataset. + + Returns: + x (dict): video clip with updated label information. + """ + label = ( + utils.aggregate_labels(x["label"]) + if mode == "train" + else x["video_label"] + ) + x["label"] = torch.as_tensor(utils.as_binary_vector(label, num_classes)) + + return x + + +def rgb2bgr(x): + """ + Convert clip frames from RGB mode to BRG mode. + Args: + x (Tensor): A tensor of the clip's RGB frames with shape: + (channel, time, height, width). + + Returns: + x (Tensor): Converted tensor + """ + return x[[2, 1, 0], ...] + + +@DATASET_REGISTRY.register() +def Ptvcharades(cfg, mode): + """ + Construct PyTorchVideo Charades video loader. + Load Charades data (frame paths, labels, etc. ) to Charades Dataset object. + The dataset could be downloaded from Chrades official website + (https://allenai.org/plato/charades/). + Please see datasets/DATASET.md for more information about the data format. + For `train` and `val` mode, a single clip is randomly sampled from every video + with random cropping, scaling, and flipping. For `test` mode, multiple clips are + uniformaly sampled from every video with center cropping. + Args: + cfg (CfgNode): configs. + mode (string): Options includes `train`, `val`, or `test` mode. + For the train and val mode, the data loader will take data + from the train or val set, and sample one clip per video. + For the test mode, the data loader will take data from test set, + and sample multiple clips per video. + """ + # Only support train, val, and test mode. + assert mode in [ + "train", + "val", + "test", + ], "Split '{}' not supported".format(mode) + + logger.info("Constructing Ptvcharades {}...".format(mode)) + + clip_duration = ( + (cfg.DATA.NUM_FRAMES - 1) * cfg.DATA.SAMPLING_RATE + 1 + ) / cfg.DATA.TARGET_FPS + + if mode in ["train", "val"]: + num_clips = 1 + num_crops = 1 + + transform = Compose( + [ + ApplyTransformToKey( + key="video", + transform=Compose( + [ + Lambda(div255), + NormalizeVideo(cfg.DATA.MEAN, cfg.DATA.STD), + RandomShortSideScale( + min_size=cfg.DATA.TRAIN_JITTER_SCALES[0], + max_size=cfg.DATA.TRAIN_JITTER_SCALES[1], + ), + RandomCropVideo(cfg.DATA.TRAIN_CROP_SIZE), + Lambda(rgb2bgr), + ] + + ( + [RandomHorizontalFlipVideo(p=0.5)] + if cfg.DATA.RANDOM_FLIP + else [] + ) + + [PackPathway(cfg)] + ), + ), + Lambda( + functools.partial( + process_charades_label, + mode=mode, + num_classes=cfg.MODEL.NUM_CLASSES, + ) + ), + DictToTuple(num_clips, num_crops), + ] + ) + clip_sampler = make_clip_sampler("random", clip_duration) + if cfg.NUM_GPUS > 1: + video_sampler = DistributedSampler + else: + video_sampler = ( + RandomSampler if mode == "train" else SequentialSampler + ) + else: + num_clips = cfg.TEST.NUM_ENSEMBLE_VIEWS + num_crops = cfg.TEST.NUM_SPATIAL_CROPS + + transform = Compose( + [ + ApplyTransformToKey( + key="video", + transform=Compose( + [ + Lambda(div255), + NormalizeVideo(cfg.DATA.MEAN, cfg.DATA.STD), + ShortSideScale(size=cfg.DATA.TEST_CROP_SIZE), + ] + ), + ), + UniformCropVideo(size=cfg.DATA.TEST_CROP_SIZE), + Lambda( + functools.partial( + process_charades_label, + mode=mode, + num_classes=cfg.MODEL.NUM_CLASSES, + ) + ), + ApplyTransformToKey( + key="video", + transform=Compose( + [Lambda(rgb2bgr), PackPathway(cfg)], + ), + ), + DictToTuple(num_clips, num_crops), + ] + ) + clip_sampler = make_clip_sampler( + "constant_clips_per_video", + clip_duration, + num_clips, + num_crops, + ) + video_sampler = ( + DistributedSampler if cfg.NUM_GPUS > 1 else SequentialSampler + ) + + data_path = os.path.join(cfg.DATA.PATH_TO_DATA_DIR, "{}.csv".format(mode)) + dataset = Charades( + data_path=data_path, + clip_sampler=clip_sampler, + video_sampler=video_sampler, + transform=transform, + video_path_prefix=cfg.DATA.PATH_PREFIX, + frames_per_clip=cfg.DATA.NUM_FRAMES, + ) + + logger.info( + "Constructing charades dataloader (size: {}) from {}".format( + len(dataset._path_to_videos), data_path + ) + ) + + return PTVDatasetWrapper( + num_videos=len(dataset._path_to_videos), + clips_per_video=num_clips, + crops_per_clip=num_crops, + dataset=dataset, + ) + + +@DATASET_REGISTRY.register() +def Ptvssv2(cfg, mode): + """ + Construct PyTorchVideo Something-Something v2 SSv2 video loader. + Load SSv2 data (frame paths, labels, etc. ) to SSv2 Dataset object. + The dataset could be downloaded from Chrades official website + (https://20bn.com/datasets/something-something). + Please see datasets/DATASET.md for more information about the data format. + For training and validation, a single clip is randomly sampled from every + video with random cropping and scaling. For testing, multiple clips are + uniformaly sampled from every video with uniform cropping. For uniform cropping, + we take the left, center, and right crop if the width is larger than height, + or take top, center, and bottom crop if the height is larger than the width. + Args: + cfg (CfgNode): configs. + mode (string): Options includes `train`, `val`, or `test` mode. + """ + # Only support train, val, and test mode. + assert mode in [ + "train", + "val", + "test", + ], "Split '{}' not supported".format(mode) + + logger.info("Constructing Ptvcharades {}...".format(mode)) + + if mode in ["train", "val"]: + num_clips = 1 + num_crops = 1 + + transform = Compose( + [ + ApplyTransformToKey( + key="video", + transform=Compose( + [ + Lambda(div255), + NormalizeVideo(cfg.DATA.MEAN, cfg.DATA.STD), + RandomShortSideScale( + min_size=cfg.DATA.TRAIN_JITTER_SCALES[0], + max_size=cfg.DATA.TRAIN_JITTER_SCALES[1], + ), + RandomCropVideo(cfg.DATA.TRAIN_CROP_SIZE), + Lambda(rgb2bgr), + ] + + ( + [RandomHorizontalFlipVideo(p=0.5)] + if cfg.DATA.RANDOM_FLIP + else [] + ) + + [PackPathway(cfg)] + ), + ), + DictToTuple(num_clips, num_crops), + ] + ) + clip_sampler = make_clip_sampler( + "constant_clips_per_video", + 1, # Put arbitrary duration as ssv2 always needs full video clip. + num_clips, + num_crops, + ) + if cfg.NUM_GPUS > 1: + video_sampler = DistributedSampler + else: + video_sampler = ( + RandomSampler if mode == "train" else SequentialSampler + ) + else: + assert cfg.TEST.NUM_ENSEMBLE_VIEWS == 1 + num_clips = cfg.TEST.NUM_ENSEMBLE_VIEWS + num_crops = cfg.TEST.NUM_SPATIAL_CROPS + + transform = Compose( + [ + ApplyTransformToKey( + key="video", + transform=Compose( + [ + Lambda(div255), + NormalizeVideo(cfg.DATA.MEAN, cfg.DATA.STD), + ShortSideScale(size=cfg.DATA.TEST_CROP_SIZE), + ] + ), + ), + UniformCropVideo(size=cfg.DATA.TEST_CROP_SIZE), + ApplyTransformToKey( + key="video", + transform=Compose( + [Lambda(rgb2bgr), PackPathway(cfg)], + ), + ), + DictToTuple(num_clips, num_crops), + ] + ) + clip_sampler = make_clip_sampler( + "constant_clips_per_video", + 1, # Put arbitrary duration as ssv2 always needs full video clip. + num_clips, + num_crops, + ) + video_sampler = ( + DistributedSampler if cfg.NUM_GPUS > 1 else SequentialSampler + ) + + label_name_file = os.path.join( + cfg.DATA.PATH_TO_DATA_DIR, "something-something-v2-labels.json" + ) + video_label_file = os.path.join( + cfg.DATA.PATH_TO_DATA_DIR, + "something-something-v2-{}.json".format( + "train" if mode == "train" else "validation" + ), + ) + data_path = os.path.join( + cfg.DATA.PATH_TO_DATA_DIR, + "{}.csv".format("train" if mode == "train" else "val"), + ) + dataset = SSv2( + label_name_file=label_name_file, + video_label_file=video_label_file, + video_path_label_file=data_path, + clip_sampler=clip_sampler, + video_sampler=video_sampler, + transform=transform, + video_path_prefix=cfg.DATA.PATH_PREFIX, + frames_per_clip=cfg.DATA.NUM_FRAMES, + rand_sample_frames=mode == "train", + ) + + logger.info( + "Constructing ssv2 dataloader (size: {}) from {}".format( + len(dataset._path_to_videos), data_path + ) + ) + + return PTVDatasetWrapper( + num_videos=len(dataset._path_to_videos), + clips_per_video=num_clips, + crops_per_clip=num_crops, + dataset=dataset, + ) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/rand_augment.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/rand_augment.py" new file mode 100644 index 0000000..8ee81fc --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/rand_augment.py" @@ -0,0 +1,533 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +""" +This implementation is based on +https://github.com/rwightman/pytorch-image-models/blob/master/timm/data/auto_augment.py +pulished under an Apache License 2.0. + +COMMENT FROM ORIGINAL: +AutoAugment, RandAugment, and AugMix for PyTorch +This code implements the searched ImageNet policies with various tweaks and +improvements and does not include any of the search code. AA and RA +Implementation adapted from: + https://github.com/tensorflow/tpu/blob/master/models/official/efficientnet/autoaugment.py +AugMix adapted from: + https://github.com/google-research/augmix +Papers: + AutoAugment: Learning Augmentation Policies from Data + https://arxiv.org/abs/1805.09501 + Learning Data Augmentation Strategies for Object Detection + https://arxiv.org/abs/1906.11172 + RandAugment: Practical automated data augmentation... + https://arxiv.org/abs/1909.13719 + AugMix: A Simple Data Processing Method to Improve Robustness and + Uncertainty https://arxiv.org/abs/1912.02781 + +Hacked together by / Copyright 2020 Ross Wightman +""" + +import math +import numpy as np +import random +import re +import PIL +from PIL import Image, ImageEnhance, ImageOps + +_PIL_VER = tuple([int(x) for x in PIL.__version__.split(".")[:2]]) + +_FILL = (128, 128, 128) + +# This signifies the max integer that the controller RNN could predict for the +# augmentation scheme. +_MAX_LEVEL = 10.0 + +_HPARAMS_DEFAULT = { + "translate_const": 250, + "img_mean": _FILL, +} + +_RANDOM_INTERPOLATION = (Image.BILINEAR, Image.BICUBIC) + + +def _interpolation(kwargs): + interpolation = kwargs.pop("resample", Image.BILINEAR) + if isinstance(interpolation, (list, tuple)): + return random.choice(interpolation) + else: + return interpolation + + +def _check_args_tf(kwargs): + if "fillcolor" in kwargs and _PIL_VER < (5, 0): + kwargs.pop("fillcolor") + kwargs["resample"] = _interpolation(kwargs) + + +def shear_x(img, factor, **kwargs): + _check_args_tf(kwargs) + return img.transform( + img.size, Image.AFFINE, (1, factor, 0, 0, 1, 0), **kwargs + ) + + +def shear_y(img, factor, **kwargs): + _check_args_tf(kwargs) + return img.transform( + img.size, Image.AFFINE, (1, 0, 0, factor, 1, 0), **kwargs + ) + + +def translate_x_rel(img, pct, **kwargs): + pixels = pct * img.size[0] + _check_args_tf(kwargs) + return img.transform( + img.size, Image.AFFINE, (1, 0, pixels, 0, 1, 0), **kwargs + ) + + +def translate_y_rel(img, pct, **kwargs): + pixels = pct * img.size[1] + _check_args_tf(kwargs) + return img.transform( + img.size, Image.AFFINE, (1, 0, 0, 0, 1, pixels), **kwargs + ) + + +def translate_x_abs(img, pixels, **kwargs): + _check_args_tf(kwargs) + return img.transform( + img.size, Image.AFFINE, (1, 0, pixels, 0, 1, 0), **kwargs + ) + + +def translate_y_abs(img, pixels, **kwargs): + _check_args_tf(kwargs) + return img.transform( + img.size, Image.AFFINE, (1, 0, 0, 0, 1, pixels), **kwargs + ) + + +def rotate(img, degrees, **kwargs): + _check_args_tf(kwargs) + if _PIL_VER >= (5, 2): + return img.rotate(degrees, **kwargs) + elif _PIL_VER >= (5, 0): + w, h = img.size + post_trans = (0, 0) + rotn_center = (w / 2.0, h / 2.0) + angle = -math.radians(degrees) + matrix = [ + round(math.cos(angle), 15), + round(math.sin(angle), 15), + 0.0, + round(-math.sin(angle), 15), + round(math.cos(angle), 15), + 0.0, + ] + + def transform(x, y, matrix): + (a, b, c, d, e, f) = matrix + return a * x + b * y + c, d * x + e * y + f + + matrix[2], matrix[5] = transform( + -rotn_center[0] - post_trans[0], + -rotn_center[1] - post_trans[1], + matrix, + ) + matrix[2] += rotn_center[0] + matrix[5] += rotn_center[1] + return img.transform(img.size, Image.AFFINE, matrix, **kwargs) + else: + return img.rotate(degrees, resample=kwargs["resample"]) + + +def auto_contrast(img, **__): + return ImageOps.autocontrast(img) + + +def invert(img, **__): + return ImageOps.invert(img) + + +def equalize(img, **__): + return ImageOps.equalize(img) + + +def solarize(img, thresh, **__): + return ImageOps.solarize(img, thresh) + + +def solarize_add(img, add, thresh=128, **__): + lut = [] + for i in range(256): + if i < thresh: + lut.append(min(255, i + add)) + else: + lut.append(i) + if img.mode in ("L", "RGB"): + if img.mode == "RGB" and len(lut) == 256: + lut = lut + lut + lut + return img.point(lut) + else: + return img + + +def posterize(img, bits_to_keep, **__): + if bits_to_keep >= 8: + return img + return ImageOps.posterize(img, bits_to_keep) + + +def contrast(img, factor, **__): + return ImageEnhance.Contrast(img).enhance(factor) + + +def color(img, factor, **__): + return ImageEnhance.Color(img).enhance(factor) + + +def brightness(img, factor, **__): + return ImageEnhance.Brightness(img).enhance(factor) + + +def sharpness(img, factor, **__): + return ImageEnhance.Sharpness(img).enhance(factor) + + +def _randomly_negate(v): + """With 50% prob, negate the value""" + return -v if random.random() > 0.5 else v + + +def _rotate_level_to_arg(level, _hparams): + # range [-30, 30] + level = (level / _MAX_LEVEL) * 30.0 + level = _randomly_negate(level) + return (level,) + + +def _enhance_level_to_arg(level, _hparams): + # range [0.1, 1.9] + return ((level / _MAX_LEVEL) * 1.8 + 0.1,) + + +def _enhance_increasing_level_to_arg(level, _hparams): + # the 'no change' level is 1.0, moving away from that towards 0. or 2.0 increases the enhancement blend + # range [0.1, 1.9] + level = (level / _MAX_LEVEL) * 0.9 + level = 1.0 + _randomly_negate(level) + return (level,) + + +def _shear_level_to_arg(level, _hparams): + # range [-0.3, 0.3] + level = (level / _MAX_LEVEL) * 0.3 + level = _randomly_negate(level) + return (level,) + + +def _translate_abs_level_to_arg(level, hparams): + translate_const = hparams["translate_const"] + level = (level / _MAX_LEVEL) * float(translate_const) + level = _randomly_negate(level) + return (level,) + + +def _translate_rel_level_to_arg(level, hparams): + # default range [-0.45, 0.45] + translate_pct = hparams.get("translate_pct", 0.45) + level = (level / _MAX_LEVEL) * translate_pct + level = _randomly_negate(level) + return (level,) + + +def _posterize_level_to_arg(level, _hparams): + # As per Tensorflow TPU EfficientNet impl + # range [0, 4], 'keep 0 up to 4 MSB of original image' + # intensity/severity of augmentation decreases with level + return (int((level / _MAX_LEVEL) * 4),) + + +def _posterize_increasing_level_to_arg(level, hparams): + # As per Tensorflow models research and UDA impl + # range [4, 0], 'keep 4 down to 0 MSB of original image', + # intensity/severity of augmentation increases with level + return (4 - _posterize_level_to_arg(level, hparams)[0],) + + +def _posterize_original_level_to_arg(level, _hparams): + # As per original AutoAugment paper description + # range [4, 8], 'keep 4 up to 8 MSB of image' + # intensity/severity of augmentation decreases with level + return (int((level / _MAX_LEVEL) * 4) + 4,) + + +def _solarize_level_to_arg(level, _hparams): + # range [0, 256] + # intensity/severity of augmentation decreases with level + return (int((level / _MAX_LEVEL) * 256),) + + +def _solarize_increasing_level_to_arg(level, _hparams): + # range [0, 256] + # intensity/severity of augmentation increases with level + return (256 - _solarize_level_to_arg(level, _hparams)[0],) + + +def _solarize_add_level_to_arg(level, _hparams): + # range [0, 110] + return (int((level / _MAX_LEVEL) * 110),) + + +LEVEL_TO_ARG = { + "AutoContrast": None, + "Equalize": None, + "Invert": None, + "Rotate": _rotate_level_to_arg, + # There are several variations of the posterize level scaling in various Tensorflow/Google repositories/papers + "Posterize": _posterize_level_to_arg, + "PosterizeIncreasing": _posterize_increasing_level_to_arg, + "PosterizeOriginal": _posterize_original_level_to_arg, + "Solarize": _solarize_level_to_arg, + "SolarizeIncreasing": _solarize_increasing_level_to_arg, + "SolarizeAdd": _solarize_add_level_to_arg, + "Color": _enhance_level_to_arg, + "ColorIncreasing": _enhance_increasing_level_to_arg, + "Contrast": _enhance_level_to_arg, + "ContrastIncreasing": _enhance_increasing_level_to_arg, + "Brightness": _enhance_level_to_arg, + "BrightnessIncreasing": _enhance_increasing_level_to_arg, + "Sharpness": _enhance_level_to_arg, + "SharpnessIncreasing": _enhance_increasing_level_to_arg, + "ShearX": _shear_level_to_arg, + "ShearY": _shear_level_to_arg, + "TranslateX": _translate_abs_level_to_arg, + "TranslateY": _translate_abs_level_to_arg, + "TranslateXRel": _translate_rel_level_to_arg, + "TranslateYRel": _translate_rel_level_to_arg, +} + + +NAME_TO_OP = { + "AutoContrast": auto_contrast, + "Equalize": equalize, + "Invert": invert, + "Rotate": rotate, + "Posterize": posterize, + "PosterizeIncreasing": posterize, + "PosterizeOriginal": posterize, + "Solarize": solarize, + "SolarizeIncreasing": solarize, + "SolarizeAdd": solarize_add, + "Color": color, + "ColorIncreasing": color, + "Contrast": contrast, + "ContrastIncreasing": contrast, + "Brightness": brightness, + "BrightnessIncreasing": brightness, + "Sharpness": sharpness, + "SharpnessIncreasing": sharpness, + "ShearX": shear_x, + "ShearY": shear_y, + "TranslateX": translate_x_abs, + "TranslateY": translate_y_abs, + "TranslateXRel": translate_x_rel, + "TranslateYRel": translate_y_rel, +} + + +class AugmentOp: + """ + Apply for video. + """ + + def __init__(self, name, prob=0.5, magnitude=10, hparams=None): + hparams = hparams or _HPARAMS_DEFAULT + self.aug_fn = NAME_TO_OP[name] + self.level_fn = LEVEL_TO_ARG[name] + self.prob = prob + self.magnitude = magnitude + self.hparams = hparams.copy() + self.kwargs = { + "fillcolor": hparams["img_mean"] + if "img_mean" in hparams + else _FILL, + "resample": hparams["interpolation"] + if "interpolation" in hparams + else _RANDOM_INTERPOLATION, + } + + # If magnitude_std is > 0, we introduce some randomness + # in the usually fixed policy and sample magnitude from a normal distribution + # with mean `magnitude` and std-dev of `magnitude_std`. + # NOTE This is my own hack, being tested, not in papers or reference impls. + self.magnitude_std = self.hparams.get("magnitude_std", 0) + + def __call__(self, img_list): + if self.prob < 1.0 and random.random() > self.prob: + return img_list + magnitude = self.magnitude + if self.magnitude_std and self.magnitude_std > 0: + magnitude = random.gauss(magnitude, self.magnitude_std) + magnitude = min(_MAX_LEVEL, max(0, magnitude)) # clip to valid range + level_args = ( + self.level_fn(magnitude, self.hparams) + if self.level_fn is not None + else () + ) + + if isinstance(img_list, list): + return [ + self.aug_fn(img, *level_args, **self.kwargs) for img in img_list + ] + else: + return self.aug_fn(img_list, *level_args, **self.kwargs) + + +_RAND_TRANSFORMS = [ + "AutoContrast", + "Equalize", + "Invert", + "Rotate", + "Posterize", + "Solarize", + "SolarizeAdd", + "Color", + "Contrast", + "Brightness", + "Sharpness", + "ShearX", + "ShearY", + "TranslateXRel", + "TranslateYRel", +] + + +_RAND_INCREASING_TRANSFORMS = [ + "AutoContrast", + "Equalize", + "Invert", + "Rotate", + "PosterizeIncreasing", + "SolarizeIncreasing", + "SolarizeAdd", + "ColorIncreasing", + "ContrastIncreasing", + "BrightnessIncreasing", + "SharpnessIncreasing", + "ShearX", + "ShearY", + "TranslateXRel", + "TranslateYRel", +] + + +# These experimental weights are based loosely on the relative improvements mentioned in paper. +# They may not result in increased performance, but could likely be tuned to so. +_RAND_CHOICE_WEIGHTS_0 = { + "Rotate": 0.3, + "ShearX": 0.2, + "ShearY": 0.2, + "TranslateXRel": 0.1, + "TranslateYRel": 0.1, + "Color": 0.025, + "Sharpness": 0.025, + "AutoContrast": 0.025, + "Solarize": 0.005, + "SolarizeAdd": 0.005, + "Contrast": 0.005, + "Brightness": 0.005, + "Equalize": 0.005, + "Posterize": 0, + "Invert": 0, +} + + +def _select_rand_weights(weight_idx=0, transforms=None): + transforms = transforms or _RAND_TRANSFORMS + assert weight_idx == 0 # only one set of weights currently + rand_weights = _RAND_CHOICE_WEIGHTS_0 + probs = [rand_weights[k] for k in transforms] + probs /= np.sum(probs) + return probs + + +def rand_augment_ops(magnitude=10, hparams=None, transforms=None): + hparams = hparams or _HPARAMS_DEFAULT + transforms = transforms or _RAND_TRANSFORMS + return [ + AugmentOp(name, prob=0.5, magnitude=magnitude, hparams=hparams) + for name in transforms + ] + + +class RandAugment: + def __init__(self, ops, num_layers=2, choice_weights=None): + self.ops = ops + self.num_layers = num_layers + self.choice_weights = choice_weights + + def __call__(self, img): + # no replacement when using weighted choice + ops = np.random.choice( + self.ops, + self.num_layers, + replace=self.choice_weights is None, + p=self.choice_weights, + ) + for op in ops: + img = op(img) + return img + + +def rand_augment_transform(config_str, hparams): + """ + RandAugment: Practical automated data augmentation... - https://arxiv.org/abs/1909.13719 + + Create a RandAugment transform + :param config_str: String defining configuration of random augmentation. Consists of multiple sections separated by + dashes ('-'). The first section defines the specific variant of rand augment (currently only 'rand'). The remaining + sections, not order sepecific determine + 'm' - integer magnitude of rand augment + 'n' - integer num layers (number of transform ops selected per image) + 'w' - integer probabiliy weight index (index of a set of weights to influence choice of op) + 'mstd' - float std deviation of magnitude noise applied + 'inc' - integer (bool), use augmentations that increase in severity with magnitude (default: 0) + Ex 'rand-m9-n3-mstd0.5' results in RandAugment with magnitude 9, num_layers 3, magnitude_std 0.5 + 'rand-mstd1-w0' results in magnitude_std 1.0, weights 0, default magnitude of 10 and num_layers 2 + :param hparams: Other hparams (kwargs) for the RandAugmentation scheme + :return: A PyTorch compatible Transform + """ + magnitude = _MAX_LEVEL # default to _MAX_LEVEL for magnitude (currently 10) + num_layers = 2 # default to 2 ops per image + weight_idx = None # default to no probability weights for op choice + transforms = _RAND_TRANSFORMS + config = config_str.split("-") + assert config[0] == "rand" + config = config[1:] + for c in config: + cs = re.split(r"(\d.*)", c) + if len(cs) < 2: + continue + key, val = cs[:2] + if key == "mstd": + # noise param injected via hparams for now + hparams.setdefault("magnitude_std", float(val)) + elif key == "inc": + if bool(val): + transforms = _RAND_INCREASING_TRANSFORMS + elif key == "m": + magnitude = int(val) + elif key == "n": + num_layers = int(val) + elif key == "w": + weight_idx = int(val) + else: + assert NotImplementedError + ra_ops = rand_augment_ops( + magnitude=magnitude, hparams=hparams, transforms=transforms + ) + choice_weights = ( + None if weight_idx is None else _select_rand_weights(weight_idx) + ) + return RandAugment(ra_ops, num_layers, choice_weights=choice_weights) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/random_erasing.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/random_erasing.py" new file mode 100644 index 0000000..952060f --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/random_erasing.py" @@ -0,0 +1,180 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +""" +This implementation is based on +https://github.com/rwightman/pytorch-image-models/blob/master/timm/data/random_erasing.py +pulished under an Apache License 2.0. + +COMMENT FROM ORIGINAL: +Originally inspired by impl at https://github.com/zhunzhong07/Random-Erasing, Apache 2.0 +Copyright Zhun Zhong & Liang Zheng +Hacked together by / Copyright 2020 Ross Wightman +""" +import math +import random +import torch + + +def _get_pixels( + per_pixel, rand_color, patch_size, dtype=torch.float32, device="cuda" +): + # NOTE I've seen CUDA illegal memory access errors being caused by the normal_() + # paths, flip the order so normal is run on CPU if this becomes a problem + # Issue has been fixed in master https://github.com/pytorch/pytorch/issues/19508 + if per_pixel: + return torch.empty(patch_size, dtype=dtype, device=device).normal_() + elif rand_color: + return torch.empty( + (patch_size[0], 1, 1), dtype=dtype, device=device + ).normal_() + else: + return torch.zeros((patch_size[0], 1, 1), dtype=dtype, device=device) + + +class RandomErasing: + """Randomly selects a rectangle region in an image and erases its pixels. + 'Random Erasing Data Augmentation' by Zhong et al. + See https://arxiv.org/pdf/1708.04896.pdf + This variant of RandomErasing is intended to be applied to either a batch + or single image tensor after it has been normalized by dataset mean and std. + Args: + probability: Probability that the Random Erasing operation will be performed. + min_area: Minimum percentage of erased area wrt input image area. + max_area: Maximum percentage of erased area wrt input image area. + min_aspect: Minimum aspect ratio of erased area. + mode: pixel color mode, one of 'const', 'rand', or 'pixel' + 'const' - erase block is constant color of 0 for all channels + 'rand' - erase block is same per-channel random (normal) color + 'pixel' - erase block is per-pixel random (normal) color + max_count: maximum number of erasing blocks per image, area per box is scaled by count. + per-image count is randomly chosen between 1 and this value. + """ + + def __init__( + self, + probability=0.5, + min_area=0.02, + max_area=1 / 3, + min_aspect=0.3, + max_aspect=None, + mode="const", + min_count=1, + max_count=None, + num_splits=0, + device="cuda", + cube=True, + ): + self.probability = probability + self.min_area = min_area + self.max_area = max_area + max_aspect = max_aspect or 1 / min_aspect + self.log_aspect_ratio = (math.log(min_aspect), math.log(max_aspect)) + self.min_count = min_count + self.max_count = max_count or min_count + self.num_splits = num_splits + mode = mode.lower() + self.rand_color = False + self.per_pixel = False + self.cube = cube + if mode == "rand": + self.rand_color = True # per block random normal + elif mode == "pixel": + self.per_pixel = True # per pixel random normal + else: + assert not mode or mode == "const" + self.device = device + + def _erase(self, img, chan, img_h, img_w, dtype): + if random.random() > self.probability: + return + area = img_h * img_w + count = ( + self.min_count + if self.min_count == self.max_count + else random.randint(self.min_count, self.max_count) + ) + for _ in range(count): + for _ in range(10): + target_area = ( + random.uniform(self.min_area, self.max_area) * area / count + ) + aspect_ratio = math.exp(random.uniform(*self.log_aspect_ratio)) + h = int(round(math.sqrt(target_area * aspect_ratio))) + w = int(round(math.sqrt(target_area / aspect_ratio))) + if w < img_w and h < img_h: + top = random.randint(0, img_h - h) + left = random.randint(0, img_w - w) + img[:, top : top + h, left : left + w] = _get_pixels( + self.per_pixel, + self.rand_color, + (chan, h, w), + dtype=dtype, + device=self.device, + ) + break + + def _erase_cube( + self, + img, + batch_start, + batch_size, + chan, + img_h, + img_w, + dtype, + ): + if random.random() > self.probability: + return + area = img_h * img_w + count = ( + self.min_count + if self.min_count == self.max_count + else random.randint(self.min_count, self.max_count) + ) + for _ in range(count): + for _ in range(100): + target_area = ( + random.uniform(self.min_area, self.max_area) * area / count + ) + aspect_ratio = math.exp(random.uniform(*self.log_aspect_ratio)) + h = int(round(math.sqrt(target_area * aspect_ratio))) + w = int(round(math.sqrt(target_area / aspect_ratio))) + if w < img_w and h < img_h: + top = random.randint(0, img_h - h) + left = random.randint(0, img_w - w) + for i in range(batch_start, batch_size): + img_instance = img[i] + img_instance[ + :, top : top + h, left : left + w + ] = _get_pixels( + self.per_pixel, + self.rand_color, + (chan, h, w), + dtype=dtype, + device=self.device, + ) + break + + def __call__(self, input): + if len(input.size()) == 3: + self._erase(input, *input.size(), input.dtype) + else: + batch_size, chan, img_h, img_w = input.size() + # skip first slice of batch if num_splits is set (for clean portion of samples) + batch_start = ( + batch_size // self.num_splits if self.num_splits > 1 else 0 + ) + if self.cube: + self._erase_cube( + input, + batch_start, + batch_size, + chan, + img_h, + img_w, + input.dtype, + ) + else: + for i in range(batch_start, batch_size): + self._erase(input[i], chan, img_h, img_w, input.dtype) + return input diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ssv2.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ssv2.py" new file mode 100644 index 0000000..5e4c2a6 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/ssv2.py" @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import json +import numpy as np +import os +import random +from itertools import chain as chain +import torch +import torch.utils.data +from iopath.common.file_io import g_pathmgr + +import slowfast.utils.logging as logging + +from . import utils as utils +from .build import DATASET_REGISTRY + +logger = logging.get_logger(__name__) + + +@DATASET_REGISTRY.register() +class Ssv2(torch.utils.data.Dataset): + """ + Something-Something v2 (SSV2) video loader. Construct the SSV2 video loader, + then sample clips from the videos. For training and validation, a single + clip is randomly sampled from every video with random cropping, scaling, and + flipping. For testing, multiple clips are uniformaly sampled from every + video with uniform cropping. For uniform cropping, we take the left, center, + and right crop if the width is larger than height, or take top, center, and + bottom crop if the height is larger than the width. + """ + + def __init__(self, cfg, mode, num_retries=10): + """ + Load Something-Something V2 data (frame paths, labels, etc. ) to a given + Dataset object. The dataset could be downloaded from Something-Something + official website (https://20bn.com/datasets/something-something). + Please see datasets/DATASET.md for more information about the data format. + Args: + cfg (CfgNode): configs. + mode (string): Options includes `train`, `val`, or `test` mode. + For the train and val mode, the data loader will take data + from the train or val set, and sample one clip per video. + For the test mode, the data loader will take data from test set, + and sample multiple clips per video. + num_retries (int): number of retries for reading frames from disk. + """ + # Only support train, val, and test mode. + assert mode in [ + "train", + "val", + "test", + ], "Split '{}' not supported for Something-Something V2".format(mode) + self.mode = mode + self.cfg = cfg + + self._video_meta = {} + self._num_retries = num_retries + # For training or validation mode, one single clip is sampled from every + # video. For testing, NUM_ENSEMBLE_VIEWS clips are sampled from every + # video. For every clip, NUM_SPATIAL_CROPS is cropped spatially from + # the frames. + if self.mode in ["train", "val"]: + self._num_clips = 1 + elif self.mode in ["test"]: + self._num_clips = ( + cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS + ) + + logger.info("Constructing Something-Something V2 {}...".format(mode)) + self._construct_loader() + + def _construct_loader(self): + """ + Construct the video loader. + """ + # Loading label names. + with g_pathmgr.open( + os.path.join( + self.cfg.DATA.PATH_TO_DATA_DIR, + "something-something-v2-labels.json", + ), + "r", + ) as f: + label_dict = json.load(f) + + # Loading labels. + label_file = os.path.join( + self.cfg.DATA.PATH_TO_DATA_DIR, + "something-something-v2-{}.json".format( + "train" if self.mode == "train" else "validation" + ), + ) + with g_pathmgr.open(label_file, "r") as f: + label_json = json.load(f) + + self._video_names = [] + self._labels = [] + for video in label_json: + video_name = video["id"] + template = video["template"] + template = template.replace("[", "") + template = template.replace("]", "") + label = int(label_dict[template]) + self._video_names.append(video_name) + self._labels.append(label) + + path_to_file = os.path.join( + self.cfg.DATA.PATH_TO_DATA_DIR, + "{}.csv".format("train" if self.mode == "train" else "val"), + ) + assert g_pathmgr.exists(path_to_file), "{} dir not found".format( + path_to_file + ) + + self._path_to_videos, _ = utils.load_image_lists( + path_to_file, self.cfg.DATA.PATH_PREFIX + ) + + assert len(self._path_to_videos) == len(self._video_names), ( + len(self._path_to_videos), + len(self._video_names), + ) + + # From dict to list. + new_paths, new_labels = [], [] + for index in range(len(self._video_names)): + if self._video_names[index] in self._path_to_videos: + new_paths.append(self._path_to_videos[self._video_names[index]]) + new_labels.append(self._labels[index]) + + self._labels = new_labels + self._path_to_videos = new_paths + + # Extend self when self._num_clips > 1 (during testing). + self._path_to_videos = list( + chain.from_iterable( + [[x] * self._num_clips for x in self._path_to_videos] + ) + ) + self._labels = list( + chain.from_iterable([[x] * self._num_clips for x in self._labels]) + ) + self._spatial_temporal_idx = list( + chain.from_iterable( + [ + range(self._num_clips) + for _ in range(len(self._path_to_videos)) + ] + ) + ) + logger.info( + "Something-Something V2 dataloader constructed " + " (size: {}) from {}".format( + len(self._path_to_videos), path_to_file + ) + ) + + def get_seq_frames(self, index): + """ + Given the video index, return the list of sampled frame indexes. + Args: + index (int): the video index. + Returns: + seq (list): the indexes of frames of sampled from the video. + """ + num_frames = self.cfg.DATA.NUM_FRAMES + video_length = len(self._path_to_videos[index]) + + seg_size = float(video_length - 1) / num_frames + seq = [] + for i in range(num_frames): + start = int(np.round(seg_size * i)) + end = int(np.round(seg_size * (i + 1))) + if self.mode == "train": + seq.append(random.randint(start, end)) + else: + seq.append((start + end) // 2) + + return seq + + def __getitem__(self, index): + """ + Given the video index, return the list of frames, label, and video + index if the video frames can be fetched. + Args: + index (int): the video index provided by the pytorch sampler. + Returns: + frames (tensor): the frames of sampled from the video. The dimension + is `channel` x `num frames` x `height` x `width`. + label (int): the label of the current video. + index (int): the index of the video. + """ + short_cycle_idx = None + # When short cycle is used, input index is a tupple. + if isinstance(index, tuple): + index, short_cycle_idx = index + + if self.mode in ["train", "val"]: + # -1 indicates random sampling. + spatial_sample_index = -1 + min_scale = self.cfg.DATA.TRAIN_JITTER_SCALES[0] + max_scale = self.cfg.DATA.TRAIN_JITTER_SCALES[1] + crop_size = self.cfg.DATA.TRAIN_CROP_SIZE + if short_cycle_idx in [0, 1]: + crop_size = int( + round( + self.cfg.MULTIGRID.SHORT_CYCLE_FACTORS[short_cycle_idx] + * self.cfg.MULTIGRID.DEFAULT_S + ) + ) + if self.cfg.MULTIGRID.DEFAULT_S > 0: + # Decreasing the scale is equivalent to using a larger "span" + # in a sampling grid. + min_scale = int( + round( + float(min_scale) + * crop_size + / self.cfg.MULTIGRID.DEFAULT_S + ) + ) + elif self.mode in ["test"]: + # spatial_sample_index is in [0, 1, 2]. Corresponding to left, + # center, or right if width is larger than height, and top, middle, + # or bottom if height is larger than width. + spatial_sample_index = ( + self._spatial_temporal_idx[index] + % self.cfg.TEST.NUM_SPATIAL_CROPS + ) + min_scale, max_scale, crop_size = [self.cfg.DATA.TEST_CROP_SIZE] * 3 + # The testing is deterministic and no jitter should be performed. + # min_scale, max_scale, and crop_size are expect to be the same. + assert len({min_scale, max_scale, crop_size}) == 1 + else: + raise NotImplementedError( + "Does not support {} mode".format(self.mode) + ) + + label = self._labels[index] + + seq = self.get_seq_frames(index) + + frames = torch.as_tensor( + utils.retry_load_images( + [self._path_to_videos[index][frame] for frame in seq], + self._num_retries, + ) + ) + + # Perform color normalization. + frames = utils.tensor_normalize( + frames, self.cfg.DATA.MEAN, self.cfg.DATA.STD + ) + + # T H W C -> C T H W. + frames = frames.permute(3, 0, 1, 2) + # Perform data augmentation. + frames = utils.spatial_sampling( + frames, + spatial_idx=spatial_sample_index, + min_scale=min_scale, + max_scale=max_scale, + crop_size=crop_size, + random_horizontal_flip=self.cfg.DATA.RANDOM_FLIP, + inverse_uniform_sampling=self.cfg.DATA.INV_UNIFORM_SAMPLE, + ) + frames = utils.pack_pathway_output(self.cfg, frames) + return frames, label, index, {} + + def __len__(self): + """ + Returns: + (int): the number of videos in the dataset. + """ + return self.num_videos + + @property + def num_videos(self): + """ + Returns: + (int): the number of videos in the dataset. + """ + return len(self._path_to_videos) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/transform.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/transform.py" new file mode 100644 index 0000000..0fe0267 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/transform.py" @@ -0,0 +1,894 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import math +import numpy as np + +# import cv2 +import random +import torch +import torchvision.transforms.functional as F +from PIL import Image +from torchvision import transforms + +from .rand_augment import rand_augment_transform +from .random_erasing import RandomErasing + +_pil_interpolation_to_str = { + Image.NEAREST: "PIL.Image.NEAREST", + Image.BILINEAR: "PIL.Image.BILINEAR", + Image.BICUBIC: "PIL.Image.BICUBIC", + Image.LANCZOS: "PIL.Image.LANCZOS", + Image.HAMMING: "PIL.Image.HAMMING", + Image.BOX: "PIL.Image.BOX", +} + + +_RANDOM_INTERPOLATION = (Image.BILINEAR, Image.BICUBIC) + + +def _pil_interp(method): + if method == "bicubic": + return Image.BICUBIC + elif method == "lanczos": + return Image.LANCZOS + elif method == "hamming": + return Image.HAMMING + else: + return Image.BILINEAR + + +def random_short_side_scale_jitter( + images, min_size, max_size, boxes=None, inverse_uniform_sampling=False +): + """ + Perform a spatial short scale jittering on the given images and + corresponding boxes. + Args: + images (tensor): images to perform scale jitter. Dimension is + `num frames` x `channel` x `height` x `width`. + min_size (int): the minimal size to scale the frames. + max_size (int): the maximal size to scale the frames. + boxes (ndarray): optional. Corresponding boxes to images. + Dimension is `num boxes` x 4. + inverse_uniform_sampling (bool): if True, sample uniformly in + [1 / max_scale, 1 / min_scale] and take a reciprocal to get the + scale. If False, take a uniform sample from [min_scale, max_scale]. + Returns: + (tensor): the scaled images with dimension of + `num frames` x `channel` x `new height` x `new width`. + (ndarray or None): the scaled boxes with dimension of + `num boxes` x 4. + """ + if inverse_uniform_sampling: + size = int( + round(1.0 / np.random.uniform(1.0 / max_size, 1.0 / min_size)) + ) + else: + size = int(round(np.random.uniform(min_size, max_size))) + + height = images.shape[2] + width = images.shape[3] + if (width <= height and width == size) or ( + height <= width and height == size + ): + return images, boxes + new_width = size + new_height = size + if width < height: + new_height = int(math.floor((float(height) / width) * size)) + if boxes is not None: + boxes = boxes * float(new_height) / height + else: + new_width = int(math.floor((float(width) / height) * size)) + if boxes is not None: + boxes = boxes * float(new_width) / width + + return ( + torch.nn.functional.interpolate( + images, + size=(new_height, new_width), + mode="bilinear", + align_corners=False, + ), + boxes, + ) + + +def crop_boxes(boxes, x_offset, y_offset): + """ + Peform crop on the bounding boxes given the offsets. + Args: + boxes (ndarray or None): bounding boxes to peform crop. The dimension + is `num boxes` x 4. + x_offset (int): cropping offset in the x axis. + y_offset (int): cropping offset in the y axis. + Returns: + cropped_boxes (ndarray or None): the cropped boxes with dimension of + `num boxes` x 4. + """ + cropped_boxes = boxes.copy() + cropped_boxes[:, [0, 2]] = boxes[:, [0, 2]] - x_offset + cropped_boxes[:, [1, 3]] = boxes[:, [1, 3]] - y_offset + + return cropped_boxes + + +def random_crop(images, size, boxes=None): + """ + Perform random spatial crop on the given images and corresponding boxes. + Args: + images (tensor): images to perform random crop. The dimension is + `num frames` x `channel` x `height` x `width`. + size (int): the size of height and width to crop on the image. + boxes (ndarray or None): optional. Corresponding boxes to images. + Dimension is `num boxes` x 4. + Returns: + cropped (tensor): cropped images with dimension of + `num frames` x `channel` x `size` x `size`. + cropped_boxes (ndarray or None): the cropped boxes with dimension of + `num boxes` x 4. + """ + if images.shape[2] == size and images.shape[3] == size: + return images + height = images.shape[2] + width = images.shape[3] + y_offset = 0 + if height > size: + y_offset = int(np.random.randint(0, height - size)) + x_offset = 0 + if width > size: + x_offset = int(np.random.randint(0, width - size)) + cropped = images[ + :, :, y_offset : y_offset + size, x_offset : x_offset + size + ] + + cropped_boxes = ( + crop_boxes(boxes, x_offset, y_offset) if boxes is not None else None + ) + + return cropped, cropped_boxes + + +def horizontal_flip(prob, images, boxes=None): + """ + Perform horizontal flip on the given images and corresponding boxes. + Args: + prob (float): probility to flip the images. + images (tensor): images to perform horizontal flip, the dimension is + `num frames` x `channel` x `height` x `width`. + boxes (ndarray or None): optional. Corresponding boxes to images. + Dimension is `num boxes` x 4. + Returns: + images (tensor): images with dimension of + `num frames` x `channel` x `height` x `width`. + flipped_boxes (ndarray or None): the flipped boxes with dimension of + `num boxes` x 4. + """ + if boxes is None: + flipped_boxes = None + else: + flipped_boxes = boxes.copy() + + if np.random.uniform() < prob: + images = images.flip((-1)) + + if len(images.shape) == 3: + width = images.shape[2] + elif len(images.shape) == 4: + width = images.shape[3] + else: + raise NotImplementedError("Dimension does not supported") + if boxes is not None: + flipped_boxes[:, [0, 2]] = width - boxes[:, [2, 0]] - 1 + + return images, flipped_boxes + + +def uniform_crop(images, size, spatial_idx, boxes=None, scale_size=None): + """ + Perform uniform spatial sampling on the images and corresponding boxes. + Args: + images (tensor): images to perform uniform crop. The dimension is + `num frames` x `channel` x `height` x `width`. + size (int): size of height and weight to crop the images. + spatial_idx (int): 0, 1, or 2 for left, center, and right crop if width + is larger than height. Or 0, 1, or 2 for top, center, and bottom + crop if height is larger than width. + boxes (ndarray or None): optional. Corresponding boxes to images. + Dimension is `num boxes` x 4. + scale_size (int): optinal. If not None, resize the images to scale_size before + performing any crop. + Returns: + cropped (tensor): images with dimension of + `num frames` x `channel` x `size` x `size`. + cropped_boxes (ndarray or None): the cropped boxes with dimension of + `num boxes` x 4. + """ + assert spatial_idx in [0, 1, 2] + ndim = len(images.shape) + if ndim == 3: + images = images.unsqueeze(0) + height = images.shape[2] + width = images.shape[3] + + if scale_size is not None: + if width <= height: + width, height = scale_size, int(height / width * scale_size) + else: + width, height = int(width / height * scale_size), scale_size + images = torch.nn.functional.interpolate( + images, + size=(height, width), + mode="bilinear", + align_corners=False, + ) + + y_offset = int(math.ceil((height - size) / 2)) + x_offset = int(math.ceil((width - size) / 2)) + + if height > width: + if spatial_idx == 0: + y_offset = 0 + elif spatial_idx == 2: + y_offset = height - size + else: + if spatial_idx == 0: + x_offset = 0 + elif spatial_idx == 2: + x_offset = width - size + cropped = images[ + :, :, y_offset : y_offset + size, x_offset : x_offset + size + ] + cropped_boxes = ( + crop_boxes(boxes, x_offset, y_offset) if boxes is not None else None + ) + if ndim == 3: + cropped = cropped.squeeze(0) + return cropped, cropped_boxes + + +def clip_boxes_to_image(boxes, height, width): + """ + Clip an array of boxes to an image with the given height and width. + Args: + boxes (ndarray): bounding boxes to perform clipping. + Dimension is `num boxes` x 4. + height (int): given image height. + width (int): given image width. + Returns: + clipped_boxes (ndarray): the clipped boxes with dimension of + `num boxes` x 4. + """ + clipped_boxes = boxes.copy() + clipped_boxes[:, [0, 2]] = np.minimum( + width - 1.0, np.maximum(0.0, boxes[:, [0, 2]]) + ) + clipped_boxes[:, [1, 3]] = np.minimum( + height - 1.0, np.maximum(0.0, boxes[:, [1, 3]]) + ) + return clipped_boxes + + +def blend(images1, images2, alpha): + """ + Blend two images with a given weight alpha. + Args: + images1 (tensor): the first images to be blended, the dimension is + `num frames` x `channel` x `height` x `width`. + images2 (tensor): the second images to be blended, the dimension is + `num frames` x `channel` x `height` x `width`. + alpha (float): the blending weight. + Returns: + (tensor): blended images, the dimension is + `num frames` x `channel` x `height` x `width`. + """ + return images1 * alpha + images2 * (1 - alpha) + + +def grayscale(images): + """ + Get the grayscale for the input images. The channels of images should be + in order BGR. + Args: + images (tensor): the input images for getting grayscale. Dimension is + `num frames` x `channel` x `height` x `width`. + Returns: + img_gray (tensor): blended images, the dimension is + `num frames` x `channel` x `height` x `width`. + """ + # R -> 0.299, G -> 0.587, B -> 0.114. + img_gray = torch.tensor(images) + gray_channel = ( + 0.299 * images[:, 2] + 0.587 * images[:, 1] + 0.114 * images[:, 0] + ) + img_gray[:, 0] = gray_channel + img_gray[:, 1] = gray_channel + img_gray[:, 2] = gray_channel + return img_gray + + +def color_jitter(images, img_brightness=0, img_contrast=0, img_saturation=0): + """ + Perfrom a color jittering on the input images. The channels of images + should be in order BGR. + Args: + images (tensor): images to perform color jitter. Dimension is + `num frames` x `channel` x `height` x `width`. + img_brightness (float): jitter ratio for brightness. + img_contrast (float): jitter ratio for contrast. + img_saturation (float): jitter ratio for saturation. + Returns: + images (tensor): the jittered images, the dimension is + `num frames` x `channel` x `height` x `width`. + """ + + jitter = [] + if img_brightness != 0: + jitter.append("brightness") + if img_contrast != 0: + jitter.append("contrast") + if img_saturation != 0: + jitter.append("saturation") + + if len(jitter) > 0: + order = np.random.permutation(np.arange(len(jitter))) + for idx in range(0, len(jitter)): + if jitter[order[idx]] == "brightness": + images = brightness_jitter(img_brightness, images) + elif jitter[order[idx]] == "contrast": + images = contrast_jitter(img_contrast, images) + elif jitter[order[idx]] == "saturation": + images = saturation_jitter(img_saturation, images) + return images + + +def brightness_jitter(var, images): + """ + Perfrom brightness jittering on the input images. The channels of images + should be in order BGR. + Args: + var (float): jitter ratio for brightness. + images (tensor): images to perform color jitter. Dimension is + `num frames` x `channel` x `height` x `width`. + Returns: + images (tensor): the jittered images, the dimension is + `num frames` x `channel` x `height` x `width`. + """ + alpha = 1.0 + np.random.uniform(-var, var) + + img_bright = torch.zeros(images.shape) + images = blend(images, img_bright, alpha) + return images + + +def contrast_jitter(var, images): + """ + Perfrom contrast jittering on the input images. The channels of images + should be in order BGR. + Args: + var (float): jitter ratio for contrast. + images (tensor): images to perform color jitter. Dimension is + `num frames` x `channel` x `height` x `width`. + Returns: + images (tensor): the jittered images, the dimension is + `num frames` x `channel` x `height` x `width`. + """ + alpha = 1.0 + np.random.uniform(-var, var) + + img_gray = grayscale(images) + img_gray[:] = torch.mean(img_gray, dim=(1, 2, 3), keepdim=True) + images = blend(images, img_gray, alpha) + return images + + +def saturation_jitter(var, images): + """ + Perfrom saturation jittering on the input images. The channels of images + should be in order BGR. + Args: + var (float): jitter ratio for saturation. + images (tensor): images to perform color jitter. Dimension is + `num frames` x `channel` x `height` x `width`. + Returns: + images (tensor): the jittered images, the dimension is + `num frames` x `channel` x `height` x `width`. + """ + alpha = 1.0 + np.random.uniform(-var, var) + img_gray = grayscale(images) + images = blend(images, img_gray, alpha) + + return images + + +def lighting_jitter(images, alphastd, eigval, eigvec): + """ + Perform AlexNet-style PCA jitter on the given images. + Args: + images (tensor): images to perform lighting jitter. Dimension is + `num frames` x `channel` x `height` x `width`. + alphastd (float): jitter ratio for PCA jitter. + eigval (list): eigenvalues for PCA jitter. + eigvec (list[list]): eigenvectors for PCA jitter. + Returns: + out_images (tensor): the jittered images, the dimension is + `num frames` x `channel` x `height` x `width`. + """ + if alphastd == 0: + return images + # generate alpha1, alpha2, alpha3. + alpha = np.random.normal(0, alphastd, size=(1, 3)) + eig_vec = np.array(eigvec) + eig_val = np.reshape(eigval, (1, 3)) + rgb = np.sum( + eig_vec * np.repeat(alpha, 3, axis=0) * np.repeat(eig_val, 3, axis=0), + axis=1, + ) + out_images = torch.zeros_like(images) + if len(images.shape) == 3: + # C H W + channel_dim = 0 + elif len(images.shape) == 4: + # T C H W + channel_dim = 1 + else: + raise NotImplementedError(f"Unsupported dimension {len(images.shape)}") + + for idx in range(images.shape[channel_dim]): + # C H W + if len(images.shape) == 3: + out_images[idx] = images[idx] + rgb[2 - idx] + # T C H W + elif len(images.shape) == 4: + out_images[:, idx] = images[:, idx] + rgb[2 - idx] + else: + raise NotImplementedError( + f"Unsupported dimension {len(images.shape)}" + ) + + return out_images + + +def color_normalization(images, mean, stddev): + """ + Perform color nomration on the given images. + Args: + images (tensor): images to perform color normalization. Dimension is + `num frames` x `channel` x `height` x `width`. + mean (list): mean values for normalization. + stddev (list): standard deviations for normalization. + + Returns: + out_images (tensor): the noramlized images, the dimension is + `num frames` x `channel` x `height` x `width`. + """ + if len(images.shape) == 3: + assert ( + len(mean) == images.shape[0] + ), "channel mean not computed properly" + assert ( + len(stddev) == images.shape[0] + ), "channel stddev not computed properly" + elif len(images.shape) == 4: + assert ( + len(mean) == images.shape[1] + ), "channel mean not computed properly" + assert ( + len(stddev) == images.shape[1] + ), "channel stddev not computed properly" + else: + raise NotImplementedError(f"Unsupported dimension {len(images.shape)}") + + out_images = torch.zeros_like(images) + for idx in range(len(mean)): + # C H W + if len(images.shape) == 3: + out_images[idx] = (images[idx] - mean[idx]) / stddev[idx] + elif len(images.shape) == 4: + out_images[:, idx] = (images[:, idx] - mean[idx]) / stddev[idx] + else: + raise NotImplementedError( + f"Unsupported dimension {len(images.shape)}" + ) + return out_images + + +def _get_param_spatial_crop( + scale, ratio, height, width, num_repeat=10, log_scale=True, switch_hw=False +): + """ + Given scale, ratio, height and width, return sampled coordinates of the videos. + """ + for _ in range(num_repeat): + area = height * width + target_area = random.uniform(*scale) * area + if log_scale: + log_ratio = (math.log(ratio[0]), math.log(ratio[1])) + aspect_ratio = math.exp(random.uniform(*log_ratio)) + else: + aspect_ratio = random.uniform(*ratio) + + w = int(round(math.sqrt(target_area * aspect_ratio))) + h = int(round(math.sqrt(target_area / aspect_ratio))) + + if np.random.uniform() < 0.5 and switch_hw: + w, h = h, w + + if 0 < w <= width and 0 < h <= height: + i = random.randint(0, height - h) + j = random.randint(0, width - w) + return i, j, h, w + + # Fallback to central crop + in_ratio = float(width) / float(height) + if in_ratio < min(ratio): + w = width + h = int(round(w / min(ratio))) + elif in_ratio > max(ratio): + h = height + w = int(round(h * max(ratio))) + else: # whole image + w = width + h = height + i = (height - h) // 2 + j = (width - w) // 2 + return i, j, h, w + + +def random_resized_crop( + images, + target_height, + target_width, + scale=(0.8, 1.0), + ratio=(3.0 / 4.0, 4.0 / 3.0), +): + """ + Crop the given images to random size and aspect ratio. A crop of random + size (default: of 0.08 to 1.0) of the original size and a random aspect + ratio (default: of 3/4 to 4/3) of the original aspect ratio is made. This + crop is finally resized to given size. This is popularly used to train the + Inception networks. + + Args: + images: Images to perform resizing and cropping. + target_height: Desired height after cropping. + target_width: Desired width after cropping. + scale: Scale range of Inception-style area based random resizing. + ratio: Aspect ratio range of Inception-style area based random resizing. + """ + + height = images.shape[2] + width = images.shape[3] + + i, j, h, w = _get_param_spatial_crop(scale, ratio, height, width) + cropped = images[:, :, i : i + h, j : j + w] + return torch.nn.functional.interpolate( + cropped, + size=(target_height, target_width), + mode="bilinear", + align_corners=False, + ) + + +def random_resized_crop_with_shift( + images, + target_height, + target_width, + scale=(0.8, 1.0), + ratio=(3.0 / 4.0, 4.0 / 3.0), +): + """ + This is similar to random_resized_crop. However, it samples two different + boxes (for cropping) for the first and last frame. It then linearly + interpolates the two boxes for other frames. + + Args: + images: Images to perform resizing and cropping. + target_height: Desired height after cropping. + target_width: Desired width after cropping. + scale: Scale range of Inception-style area based random resizing. + ratio: Aspect ratio range of Inception-style area based random resizing. + """ + t = images.shape[1] + height = images.shape[2] + width = images.shape[3] + + i, j, h, w = _get_param_spatial_crop(scale, ratio, height, width) + i_, j_, h_, w_ = _get_param_spatial_crop(scale, ratio, height, width) + i_s = [int(i) for i in torch.linspace(i, i_, steps=t).tolist()] + j_s = [int(i) for i in torch.linspace(j, j_, steps=t).tolist()] + h_s = [int(i) for i in torch.linspace(h, h_, steps=t).tolist()] + w_s = [int(i) for i in torch.linspace(w, w_, steps=t).tolist()] + out = torch.zeros((3, t, target_height, target_width)) + for ind in range(t): + out[:, ind : ind + 1, :, :] = torch.nn.functional.interpolate( + images[ + :, + ind : ind + 1, + i_s[ind] : i_s[ind] + h_s[ind], + j_s[ind] : j_s[ind] + w_s[ind], + ], + size=(target_height, target_width), + mode="bilinear", + align_corners=False, + ) + return out + + +def create_random_augment( + input_size, + auto_augment=None, + interpolation="bilinear", +): + """ + Get video randaug transform. + + Args: + input_size: The size of the input video in tuple. + auto_augment: Parameters for randaug. An example: + "rand-m7-n4-mstd0.5-inc1" (m is the magnitude and n is the number + of operations to apply). + interpolation: Interpolation method. + """ + if isinstance(input_size, tuple): + img_size = input_size[-2:] + else: + img_size = input_size + + if auto_augment: + assert isinstance(auto_augment, str) + if isinstance(img_size, tuple): + img_size_min = min(img_size) + else: + img_size_min = img_size + aa_params = {"translate_const": int(img_size_min * 0.45)} + if interpolation and interpolation != "random": + aa_params["interpolation"] = _pil_interp(interpolation) + if auto_augment.startswith("rand"): + return transforms.Compose( + [rand_augment_transform(auto_augment, aa_params)] + ) + raise NotImplementedError + + +def random_sized_crop_img( + im, + size, + jitter_scale=(0.08, 1.0), + jitter_aspect=(3.0 / 4.0, 4.0 / 3.0), + max_iter=10, +): + """ + Performs Inception-style cropping (used for training). + """ + assert ( + len(im.shape) == 3 + ), "Currently only support image for random_sized_crop" + h, w = im.shape[1:3] + i, j, h, w = _get_param_spatial_crop( + scale=jitter_scale, + ratio=jitter_aspect, + height=h, + width=w, + num_repeat=max_iter, + log_scale=False, + switch_hw=True, + ) + cropped = im[:, i : i + h, j : j + w] + return torch.nn.functional.interpolate( + cropped.unsqueeze(0), + size=(size, size), + mode="bilinear", + align_corners=False, + ).squeeze(0) + + +# The following code are modified based on timm lib, we will replace the following +# contents with dependency from PyTorchVideo. +# https://github.com/facebookresearch/pytorchvideo +class RandomResizedCropAndInterpolation: + """Crop the given PIL Image to random size and aspect ratio with random interpolation. + A crop of random size (default: of 0.08 to 1.0) of the original size and a random + aspect ratio (default: of 3/4 to 4/3) of the original aspect ratio is made. This crop + is finally resized to given size. + This is popularly used to train the Inception networks. + Args: + size: expected output size of each edge + scale: range of size of the origin size cropped + ratio: range of aspect ratio of the origin aspect ratio cropped + interpolation: Default: PIL.Image.BILINEAR + """ + + def __init__( + self, + size, + scale=(0.08, 1.0), + ratio=(3.0 / 4.0, 4.0 / 3.0), + interpolation="bilinear", + ): + if isinstance(size, tuple): + self.size = size + else: + self.size = (size, size) + if (scale[0] > scale[1]) or (ratio[0] > ratio[1]): + print("range should be of kind (min, max)") + + if interpolation == "random": + self.interpolation = _RANDOM_INTERPOLATION + else: + self.interpolation = _pil_interp(interpolation) + self.scale = scale + self.ratio = ratio + + @staticmethod + def get_params(img, scale, ratio): + """Get parameters for ``crop`` for a random sized crop. + Args: + img (PIL Image): Image to be cropped. + scale (tuple): range of size of the origin size cropped + ratio (tuple): range of aspect ratio of the origin aspect ratio cropped + Returns: + tuple: params (i, j, h, w) to be passed to ``crop`` for a random + sized crop. + """ + area = img.size[0] * img.size[1] + + for _ in range(10): + target_area = random.uniform(*scale) * area + log_ratio = (math.log(ratio[0]), math.log(ratio[1])) + aspect_ratio = math.exp(random.uniform(*log_ratio)) + + w = int(round(math.sqrt(target_area * aspect_ratio))) + h = int(round(math.sqrt(target_area / aspect_ratio))) + + if w <= img.size[0] and h <= img.size[1]: + i = random.randint(0, img.size[1] - h) + j = random.randint(0, img.size[0] - w) + return i, j, h, w + + # Fallback to central crop + in_ratio = img.size[0] / img.size[1] + if in_ratio < min(ratio): + w = img.size[0] + h = int(round(w / min(ratio))) + elif in_ratio > max(ratio): + h = img.size[1] + w = int(round(h * max(ratio))) + else: # whole image + w = img.size[0] + h = img.size[1] + i = (img.size[1] - h) // 2 + j = (img.size[0] - w) // 2 + return i, j, h, w + + def __call__(self, img): + """ + Args: + img (PIL Image): Image to be cropped and resized. + Returns: + PIL Image: Randomly cropped and resized image. + """ + i, j, h, w = self.get_params(img, self.scale, self.ratio) + if isinstance(self.interpolation, (tuple, list)): + interpolation = random.choice(self.interpolation) + else: + interpolation = self.interpolation + return F.resized_crop(img, i, j, h, w, self.size, interpolation) + + def __repr__(self): + if isinstance(self.interpolation, (tuple, list)): + interpolate_str = " ".join( + [_pil_interpolation_to_str[x] for x in self.interpolation] + ) + else: + interpolate_str = _pil_interpolation_to_str[self.interpolation] + format_string = self.__class__.__name__ + "(size={0}".format(self.size) + format_string += ", scale={0}".format( + tuple(round(s, 4) for s in self.scale) + ) + format_string += ", ratio={0}".format( + tuple(round(r, 4) for r in self.ratio) + ) + format_string += ", interpolation={0})".format(interpolate_str) + return format_string + + +def transforms_imagenet_train( + img_size=224, + scale=None, + ratio=None, + hflip=0.5, + vflip=0.0, + color_jitter=0.4, + auto_augment=None, + interpolation="random", + use_prefetcher=False, + mean=(0.485, 0.456, 0.406), + std=(0.229, 0.224, 0.225), + re_prob=0.0, + re_mode="const", + re_count=1, + re_num_splits=0, + separate=False, +): + """ + If separate==True, the transforms are returned as a tuple of 3 separate transforms + for use in a mixing dataset that passes + * all data through the first (primary) transform, called the 'clean' data + * a portion of the data through the secondary transform + * normalizes and converts the branches above with the third, final transform + """ + if isinstance(img_size, tuple): + img_size = img_size[-2:] + else: + img_size = img_size + + scale = tuple(scale or (0.08, 1.0)) # default imagenet scale range + ratio = tuple( + ratio or (3.0 / 4.0, 4.0 / 3.0) + ) # default imagenet ratio range + primary_tfl = [ + RandomResizedCropAndInterpolation( + img_size, scale=scale, ratio=ratio, interpolation=interpolation + ) + ] + if hflip > 0.0: + primary_tfl += [transforms.RandomHorizontalFlip(p=hflip)] + if vflip > 0.0: + primary_tfl += [transforms.RandomVerticalFlip(p=vflip)] + + secondary_tfl = [] + if auto_augment: + assert isinstance(auto_augment, str) + if isinstance(img_size, tuple): + img_size_min = min(img_size) + else: + img_size_min = img_size + aa_params = dict( + translate_const=int(img_size_min * 0.45), + img_mean=tuple([min(255, round(255 * x)) for x in mean]), + ) + if interpolation and interpolation != "random": + aa_params["interpolation"] = _pil_interp(interpolation) + if auto_augment.startswith("rand"): + secondary_tfl += [rand_augment_transform(auto_augment, aa_params)] + elif auto_augment.startswith("augmix"): + raise NotImplementedError("Augmix not implemented") + else: + raise NotImplementedError("Auto aug not implemented") + elif color_jitter is not None: + # color jitter is enabled when not using AA + if isinstance(color_jitter, (list, tuple)): + # color jitter should be a 3-tuple/list if spec brightness/contrast/saturation + # or 4 if also augmenting hue + assert len(color_jitter) in (3, 4) + else: + # if it's a scalar, duplicate for brightness, contrast, and saturation, no hue + color_jitter = (float(color_jitter),) * 3 + secondary_tfl += [transforms.ColorJitter(*color_jitter)] + + final_tfl = [] + final_tfl += [ + transforms.ToTensor(), + transforms.Normalize(mean=torch.tensor(mean), std=torch.tensor(std)), + ] + if re_prob > 0.0: + final_tfl.append( + RandomErasing( + re_prob, + mode=re_mode, + max_count=re_count, + num_splits=re_num_splits, + device="cpu", + cube=False, + ) + ) + + if separate: + return ( + transforms.Compose(primary_tfl), + transforms.Compose(secondary_tfl), + transforms.Compose(final_tfl), + ) + else: + return transforms.Compose(primary_tfl + secondary_tfl + final_tfl) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/utils.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/utils.py" new file mode 100644 index 0000000..47a1b7f --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/utils.py" @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 + +import logging +import numpy as np +import os +import random +import time +from collections import defaultdict +import cv2 +import torch +from iopath.common.file_io import g_pathmgr +from torch.utils.data.distributed import DistributedSampler + +from . import transform as transform + +logger = logging.getLogger(__name__) + + +def retry_load_images(image_paths, retry=10, backend="pytorch"): + """ + This function is to load images with support of retrying for failed load. + + Args: + image_paths (list): paths of images needed to be loaded. + retry (int, optional): maximum time of loading retrying. Defaults to 10. + backend (str): `pytorch` or `cv2`. + + Returns: + imgs (list): list of loaded images. + """ + for i in range(retry): + imgs = [] + for image_path in image_paths: + with g_pathmgr.open(image_path, "rb") as f: + img_str = np.frombuffer(f.read(), np.uint8) + img = cv2.imdecode(img_str, flags=cv2.IMREAD_COLOR) + imgs.append(img) + + if all(img is not None for img in imgs): + if backend == "pytorch": + imgs = torch.as_tensor(np.stack(imgs)) + return imgs + else: + logger.warn("Reading failed. Will retry.") + time.sleep(1.0) + if i == retry - 1: + raise Exception("Failed to load images {}".format(image_paths)) + + +def get_sequence(center_idx, half_len, sample_rate, num_frames): + """ + Sample frames among the corresponding clip. + + Args: + center_idx (int): center frame idx for current clip + half_len (int): half of the clip length + sample_rate (int): sampling rate for sampling frames inside of the clip + num_frames (int): number of expected sampled frames + + Returns: + seq (list): list of indexes of sampled frames in this clip. + """ + seq = list(range(center_idx - half_len, center_idx + half_len, sample_rate)) + + for seq_idx in range(len(seq)): + if seq[seq_idx] < 0: + seq[seq_idx] = 0 + elif seq[seq_idx] >= num_frames: + seq[seq_idx] = num_frames - 1 + return seq + + +def pack_pathway_output(cfg, frames): + """ + Prepare output as a list of tensors. Each tensor corresponding to a + unique pathway. + Args: + frames (tensor): frames of images sampled from the video. The + dimension is `channel` x `num frames` x `height` x `width`. + Returns: + frame_list (list): list of tensors with the dimension of + `channel` x `num frames` x `height` x `width`. + """ + if cfg.DATA.REVERSE_INPUT_CHANNEL: + frames = frames[[2, 1, 0], :, :, :] + if cfg.MODEL.ARCH in cfg.MODEL.SINGLE_PATHWAY_ARCH: + frame_list = [frames] + elif cfg.MODEL.ARCH in cfg.MODEL.MULTI_PATHWAY_ARCH: + fast_pathway = frames + # Perform temporal sampling from the fast pathway. + slow_pathway = torch.index_select( + frames, + 1, + torch.linspace( + 0, frames.shape[1] - 1, frames.shape[1] // cfg.SLOWFAST.ALPHA + ).long(), + ) + frame_list = [slow_pathway, fast_pathway] + else: + raise NotImplementedError( + "Model arch {} is not in {}".format( + cfg.MODEL.ARCH, + cfg.MODEL.SINGLE_PATHWAY_ARCH + cfg.MODEL.MULTI_PATHWAY_ARCH, + ) + ) + return frame_list + + +def spatial_sampling( + frames, + spatial_idx=-1, + min_scale=256, + max_scale=320, + crop_size=224, + random_horizontal_flip=True, + inverse_uniform_sampling=False, + aspect_ratio=None, + scale=None, + motion_shift=False, +): + """ + Perform spatial sampling on the given video frames. If spatial_idx is + -1, perform random scale, random crop, and random flip on the given + frames. If spatial_idx is 0, 1, or 2, perform spatial uniform sampling + with the given spatial_idx. + Args: + frames (tensor): frames of images sampled from the video. The + dimension is `num frames` x `height` x `width` x `channel`. + spatial_idx (int): if -1, perform random spatial sampling. If 0, 1, + or 2, perform left, center, right crop if width is larger than + height, and perform top, center, buttom crop if height is larger + than width. + min_scale (int): the minimal size of scaling. + max_scale (int): the maximal size of scaling. + crop_size (int): the size of height and width used to crop the + frames. + inverse_uniform_sampling (bool): if True, sample uniformly in + [1 / max_scale, 1 / min_scale] and take a reciprocal to get the + scale. If False, take a uniform sample from [min_scale, + max_scale]. + aspect_ratio (list): Aspect ratio range for resizing. + scale (list): Scale range for resizing. + motion_shift (bool): Whether to apply motion shift for resizing. + Returns: + frames (tensor): spatially sampled frames. + """ + assert spatial_idx in [-1, 0, 1, 2] + if spatial_idx == -1: + if aspect_ratio is None and scale is None: + frames, _ = transform.random_short_side_scale_jitter( + images=frames, + min_size=min_scale, + max_size=max_scale, + inverse_uniform_sampling=inverse_uniform_sampling, + ) + frames, _ = transform.random_crop(frames, crop_size) + else: + transform_func = ( + transform.random_resized_crop_with_shift + if motion_shift + else transform.random_resized_crop + ) + frames = transform_func( + images=frames, + target_height=crop_size, + target_width=crop_size, + scale=scale, + ratio=aspect_ratio, + ) + if random_horizontal_flip: + frames, _ = transform.horizontal_flip(0.5, frames) + else: + # The testing is deterministic and no jitter should be performed. + # min_scale, max_scale, and crop_size are expect to be the same. + assert len({min_scale, max_scale, crop_size}) == 1 + frames, _ = transform.random_short_side_scale_jitter( + frames, min_scale, max_scale + ) + frames, _ = transform.uniform_crop(frames, crop_size, spatial_idx) + return frames + + +def as_binary_vector(labels, num_classes): + """ + Construct binary label vector given a list of label indices. + Args: + labels (list): The input label list. + num_classes (int): Number of classes of the label vector. + Returns: + labels (numpy array): the resulting binary vector. + """ + label_arr = np.zeros((num_classes,)) + + for lbl in set(labels): + label_arr[lbl] = 1.0 + return label_arr + + +def aggregate_labels(label_list): + """ + Join a list of label list. + Args: + labels (list): The input label list. + Returns: + labels (list): The joint list of all lists in input. + """ + all_labels = [] + for labels in label_list: + for l in labels: + all_labels.append(l) + return list(set(all_labels)) + + +def convert_to_video_level_labels(labels): + """ + Aggregate annotations from all frames of a video to form video-level labels. + Args: + labels (list): The input label list. + Returns: + labels (list): Same as input, but with each label replaced by + a video-level one. + """ + for video_id in range(len(labels)): + video_level_labels = aggregate_labels(labels[video_id]) + for i in range(len(labels[video_id])): + labels[video_id][i] = video_level_labels + return labels + + +def load_image_lists(frame_list_file, prefix="", return_list=False): + """ + Load image paths and labels from a "frame list". + Each line of the frame list contains: + `original_vido_id video_id frame_id path labels` + Args: + frame_list_file (string): path to the frame list. + prefix (str): the prefix for the path. + return_list (bool): if True, return a list. If False, return a dict. + Returns: + image_paths (list or dict): list of list containing path to each frame. + If return_list is False, then return in a dict form. + labels (list or dict): list of list containing label of each frame. + If return_list is False, then return in a dict form. + """ + image_paths = defaultdict(list) + labels = defaultdict(list) + with g_pathmgr.open(frame_list_file, "r") as f: + assert f.readline().startswith("original_vido_id") + for line in f: + row = line.split() + # original_vido_id video_id frame_id path labels + assert len(row) == 5 + video_name = row[0] + if prefix == "": + path = row[3] + else: + path = os.path.join(prefix, row[3]) + image_paths[video_name].append(path) + frame_labels = row[-1].replace('"', "") + if frame_labels != "": + labels[video_name].append( + [int(x) for x in frame_labels.split(",")] + ) + else: + labels[video_name].append([]) + + if return_list: + keys = image_paths.keys() + image_paths = [image_paths[key] for key in keys] + labels = [labels[key] for key in keys] + return image_paths, labels + return dict(image_paths), dict(labels) + + +def tensor_normalize(tensor, mean, std): + """ + Normalize a given tensor by subtracting the mean and dividing the std. + Args: + tensor (tensor): tensor to normalize. + mean (tensor or list): mean value to subtract. + std (tensor or list): std to divide. + """ + if tensor.dtype == torch.uint8: + tensor = tensor.float() + tensor = tensor / 255.0 + if type(mean) == list: + mean = torch.tensor(mean) + if type(std) == list: + std = torch.tensor(std) + tensor = tensor - mean + tensor = tensor / std + return tensor + + +def get_random_sampling_rate(long_cycle_sampling_rate, sampling_rate): + """ + When multigrid training uses a fewer number of frames, we randomly + increase the sampling rate so that some clips cover the original span. + """ + if long_cycle_sampling_rate > 0: + assert long_cycle_sampling_rate >= sampling_rate + return random.randint(sampling_rate, long_cycle_sampling_rate) + else: + return sampling_rate + + +def revert_tensor_normalize(tensor, mean, std): + """ + Revert normalization for a given tensor by multiplying by the std and adding the mean. + Args: + tensor (tensor): tensor to revert normalization. + mean (tensor or list): mean value to add. + std (tensor or list): std to multiply. + """ + if type(mean) == list: + mean = torch.tensor(mean) + if type(std) == list: + std = torch.tensor(std) + tensor = tensor * std + tensor = tensor + mean + return tensor + + +def create_sampler(dataset, shuffle, cfg): + """ + Create sampler for the given dataset. + Args: + dataset (torch.utils.data.Dataset): the given dataset. + shuffle (bool): set to ``True`` to have the data reshuffled + at every epoch. + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + Returns: + sampler (Sampler): the created sampler. + """ + sampler = DistributedSampler(dataset) if cfg.NUM_GPUS > 1 else None + + return sampler + + +def loader_worker_init_fn(dataset): + """ + Create init function passed to pytorch data loader. + Args: + dataset (torch.utils.data.Dataset): the given dataset. + """ + return None diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/video_container.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/video_container.py" new file mode 100644 index 0000000..135c776 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/datasets/video_container.py" @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import av + + +def get_video_container(path_to_vid, multi_thread_decode=False, backend="pyav"): + """ + Given the path to the video, return the pyav video container. + Args: + path_to_vid (str): path to the video. + multi_thread_decode (bool): if True, perform multi-thread decoding. + backend (str): decoder backend, options include `pyav` and + `torchvision`, default is `pyav`. + Returns: + container (container): video container. + """ + if backend == "torchvision": + with open(path_to_vid, "rb") as fp: + container = fp.read() + return container + elif backend == "pyav": + container = av.open(path_to_vid) + if multi_thread_decode: + # Enable multiple threads for decoding. + container.streams.video[0].thread_type = "AUTO" + return container + else: + raise NotImplementedError("Unknown backend {}".format(backend)) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/__init__.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/__init__.py" new file mode 100644 index 0000000..ce97190 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/__init__.py" @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +from .build import MODEL_REGISTRY, build_model # noqa +from .custom_video_model_builder import * # noqa +from .video_model_builder import ResNet, SlowFast # noqa + +try: + from .ptv_model_builder import ( + PTVCSN, + PTVX3D, + PTVR2plus1D, + PTVResNet, + PTVSlowFast, + ) # noqa +except Exception: + print("Please update your PyTorchVideo to latest master") diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/attention.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/attention.py" new file mode 100644 index 0000000..12bee23 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/attention.py" @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + + +import numpy +import torch +import torch.nn as nn + +from slowfast.models.common import DropPath, Mlp + + +def attention_pool(tensor, pool, thw_shape, has_cls_embed=True, norm=None): + if pool is None: + return tensor, thw_shape + tensor_dim = tensor.ndim + if tensor_dim == 4: + pass + elif tensor_dim == 3: + tensor = tensor.unsqueeze(1) + else: + raise NotImplementedError(f"Unsupported input dimension {tensor.shape}") + + if has_cls_embed: + cls_tok, tensor = tensor[:, :, :1, :], tensor[:, :, 1:, :] + + B, N, L, C = tensor.shape + T, H, W = thw_shape + tensor = ( + tensor.reshape(B * N, T, H, W, C).permute(0, 4, 1, 2, 3).contiguous() + ) + + tensor = pool(tensor) + + thw_shape = [tensor.shape[2], tensor.shape[3], tensor.shape[4]] + L_pooled = tensor.shape[2] * tensor.shape[3] * tensor.shape[4] + tensor = tensor.reshape(B, N, C, L_pooled).transpose(2, 3) + if has_cls_embed: + tensor = torch.cat((cls_tok, tensor), dim=2) + if norm is not None: + tensor = norm(tensor) + # Assert tensor_dim in [3, 4] + if tensor_dim == 4: + pass + else: # tensor_dim == 3: + tensor = tensor.squeeze(1) + return tensor, thw_shape + + +class MultiScaleAttention(nn.Module): + def __init__( + self, + dim, + num_heads=8, + qkv_bias=False, + drop_rate=0.0, + kernel_q=(1, 1, 1), + kernel_kv=(1, 1, 1), + stride_q=(1, 1, 1), + stride_kv=(1, 1, 1), + norm_layer=nn.LayerNorm, + has_cls_embed=True, + # Options include `conv`, `avg`, and `max`. + mode="conv", + # If True, perform pool before projection. + pool_first=False, + ): + super().__init__() + self.pool_first = pool_first + self.drop_rate = drop_rate + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = head_dim ** -0.5 + self.has_cls_embed = has_cls_embed + padding_q = [int(q // 2) for q in kernel_q] + padding_kv = [int(kv // 2) for kv in kernel_kv] + + self.q = nn.Linear(dim, dim, bias=qkv_bias) + self.k = nn.Linear(dim, dim, bias=qkv_bias) + self.v = nn.Linear(dim, dim, bias=qkv_bias) + self.proj = nn.Linear(dim, dim) + if drop_rate > 0.0: + self.proj_drop = nn.Dropout(drop_rate) + + # Skip pooling with kernel and stride size of (1, 1, 1). + if numpy.prod(kernel_q) == 1 and numpy.prod(stride_q) == 1: + kernel_q = () + if numpy.prod(kernel_kv) == 1 and numpy.prod(stride_kv) == 1: + kernel_kv = () + + if mode == "avg": + self.pool_q = ( + nn.AvgPool3d(kernel_q, stride_q, padding_q, ceil_mode=False) + if len(kernel_q) > 0 + else None + ) + self.pool_k = ( + nn.AvgPool3d(kernel_kv, stride_kv, padding_kv, ceil_mode=False) + if len(kernel_kv) > 0 + else None + ) + self.pool_v = ( + nn.AvgPool3d(kernel_kv, stride_kv, padding_kv, ceil_mode=False) + if len(kernel_kv) > 0 + else None + ) + elif mode == "max": + self.pool_q = ( + nn.MaxPool3d(kernel_q, stride_q, padding_q, ceil_mode=False) + if len(kernel_q) > 0 + else None + ) + self.pool_k = ( + nn.MaxPool3d(kernel_kv, stride_kv, padding_kv, ceil_mode=False) + if len(kernel_kv) > 0 + else None + ) + self.pool_v = ( + nn.MaxPool3d(kernel_kv, stride_kv, padding_kv, ceil_mode=False) + if len(kernel_kv) > 0 + else None + ) + elif mode == "conv": + self.pool_q = ( + nn.Conv3d( + head_dim, + head_dim, + kernel_q, + stride=stride_q, + padding=padding_q, + groups=head_dim, + bias=False, + ) + if len(kernel_q) > 0 + else None + ) + self.norm_q = norm_layer(head_dim) if len(kernel_q) > 0 else None + self.pool_k = ( + nn.Conv3d( + head_dim, + head_dim, + kernel_kv, + stride=stride_kv, + padding=padding_kv, + groups=head_dim, + bias=False, + ) + if len(kernel_kv) > 0 + else None + ) + self.norm_k = norm_layer(head_dim) if len(kernel_kv) > 0 else None + self.pool_v = ( + nn.Conv3d( + head_dim, + head_dim, + kernel_kv, + stride=stride_kv, + padding=padding_kv, + groups=head_dim, + bias=False, + ) + if len(kernel_kv) > 0 + else None + ) + self.norm_v = norm_layer(head_dim) if len(kernel_kv) > 0 else None + else: + raise NotImplementedError(f"Unsupported model {mode}") + + def forward(self, x, thw_shape): + B, N, C = x.shape + if self.pool_first: + q = k = v = x + q = self.q(q).reshape(B, N, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3) + k = self.k(k).reshape(B, N, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3) + v = self.v(v).reshape(B, N, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3) + else: + x = x.reshape(B, N, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3) + q = k = v = x + + q, q_shape = attention_pool( + q, + self.pool_q, + thw_shape, + has_cls_embed=self.has_cls_embed, + norm=self.norm_q if hasattr(self, "norm_q") else None, + ) + k, k_shape = attention_pool( + k, + self.pool_k, + thw_shape, + has_cls_embed=self.has_cls_embed, + norm=self.norm_k if hasattr(self, "norm_k") else None, + ) + v, v_shape = attention_pool( + v, + self.pool_v, + thw_shape, + has_cls_embed=self.has_cls_embed, + norm=self.norm_v if hasattr(self, "norm_v") else None, + ) + + if not self.pool_first: + if self.has_cls_embed: + q_N = numpy.prod(q_shape) + 1 + k_N = numpy.prod(k_shape) + 1 + v_N = numpy.prod(v_shape) + 1 + else: + q_N = numpy.prod(q_shape) + k_N = numpy.prod(k_shape) + v_N = numpy.prod(v_shape) + + q = q.permute(0, 2, 1, 3).reshape(B, q_N, C) + q = self.q(q).reshape(B, q_N, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3) + + v = v.permute(0, 2, 1, 3).reshape(B, v_N, C) + v = self.v(v).reshape(B, v_N, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3) + + k = k.permute(0, 2, 1, 3).reshape(B, k_N, C) + k = self.k(k).reshape(B, k_N, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3) + + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + + N = q.shape[2] + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + if self.drop_rate > 0.0: + x = self.proj_drop(x) + return x, q_shape + + +class MultiScaleBlock(nn.Module): + def __init__( + self, + dim, + dim_out, + num_heads, + mlp_ratio=4.0, + qkv_bias=False, + qk_scale=None, + drop_rate=0.0, + drop_path=0.0, + act_layer=nn.GELU, + norm_layer=nn.LayerNorm, + up_rate=None, + kernel_q=(1, 1, 1), + kernel_kv=(1, 1, 1), + stride_q=(1, 1, 1), + stride_kv=(1, 1, 1), + mode="conv", + has_cls_embed=True, + pool_first=False, + ): + super().__init__() + self.dim = dim + self.dim_out = dim_out + self.norm1 = norm_layer(dim) + kernel_skip = [s + 1 if s > 1 else s for s in stride_q] + stride_skip = stride_q + padding_skip = [int(skip // 2) for skip in kernel_skip] + self.attn = MultiScaleAttention( + dim, + num_heads=num_heads, + qkv_bias=qkv_bias, + drop_rate=drop_rate, + kernel_q=kernel_q, + kernel_kv=kernel_kv, + stride_q=stride_q, + stride_kv=stride_kv, + norm_layer=nn.LayerNorm, + has_cls_embed=has_cls_embed, + mode=mode, + pool_first=pool_first, + ) + self.drop_path = ( + DropPath(drop_path) if drop_path > 0.0 else nn.Identity() + ) + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.has_cls_embed = has_cls_embed + # TODO: check the use case for up_rate, and merge the following lines + if up_rate is not None and up_rate > 1: + mlp_dim_out = dim * up_rate + else: + mlp_dim_out = dim_out + self.mlp = Mlp( + in_features=dim, + hidden_features=mlp_hidden_dim, + out_features=mlp_dim_out, + act_layer=act_layer, + drop_rate=drop_rate, + ) + if dim != dim_out: + self.proj = nn.Linear(dim, dim_out) + + self.pool_skip = ( + nn.MaxPool3d( + kernel_skip, stride_skip, padding_skip, ceil_mode=False + ) + if len(kernel_skip) > 0 + else None + ) + + def forward(self, x, thw_shape): + x_block, thw_shape_new = self.attn(self.norm1(x), thw_shape) + x_res, _ = attention_pool( + x, self.pool_skip, thw_shape, has_cls_embed=self.has_cls_embed + ) + x = x_res + self.drop_path(x_block) + x_norm = self.norm2(x) + x_mlp = self.mlp(x_norm) + if self.dim != self.dim_out: + x = self.proj(x_norm) + x = x + self.drop_path(x_mlp) + return x, thw_shape_new diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/batchnorm_helper.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/batchnorm_helper.py" new file mode 100644 index 0000000..4e52d50 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/batchnorm_helper.py" @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""BatchNorm (BN) utility functions and custom batch-size BN implementations""" + +from functools import partial +import torch +import torch.distributed as dist +import torch.nn as nn +from torch.autograd.function import Function + +import slowfast.utils.distributed as du + + +def get_norm(cfg): + """ + Args: + cfg (CfgNode): model building configs, details are in the comments of + the config file. + Returns: + nn.Module: the normalization layer. + """ + if cfg.BN.NORM_TYPE == "batchnorm": + return nn.BatchNorm3d + elif cfg.BN.NORM_TYPE == "sub_batchnorm": + return partial(SubBatchNorm3d, num_splits=cfg.BN.NUM_SPLITS) + elif cfg.BN.NORM_TYPE == "sync_batchnorm": + return partial( + NaiveSyncBatchNorm3d, num_sync_devices=cfg.BN.NUM_SYNC_DEVICES + ) + else: + raise NotImplementedError( + "Norm type {} is not supported".format(cfg.BN.NORM_TYPE) + ) + + +class SubBatchNorm3d(nn.Module): + """ + The standard BN layer computes stats across all examples in a GPU. In some + cases it is desirable to compute stats across only a subset of examples + (e.g., in multigrid training https://arxiv.org/abs/1912.00998). + SubBatchNorm3d splits the batch dimension into N splits, and run BN on + each of them separately (so that the stats are computed on each subset of + examples (1/N of batch) independently. During evaluation, it aggregates + the stats from all splits into one BN. + """ + + def __init__(self, num_splits, **args): + """ + Args: + num_splits (int): number of splits. + args (list): other arguments. + """ + super(SubBatchNorm3d, self).__init__() + self.num_splits = num_splits + num_features = args["num_features"] + # Keep only one set of weight and bias. + if args.get("affine", True): + self.affine = True + args["affine"] = False + self.weight = torch.nn.Parameter(torch.ones(num_features)) + self.bias = torch.nn.Parameter(torch.zeros(num_features)) + else: + self.affine = False + self.bn = nn.BatchNorm3d(**args) + args["num_features"] = num_features * num_splits + self.split_bn = nn.BatchNorm3d(**args) + + def _get_aggregated_mean_std(self, means, stds, n): + """ + Calculate the aggregated mean and stds. + Args: + means (tensor): mean values. + stds (tensor): standard deviations. + n (int): number of sets of means and stds. + """ + mean = means.view(n, -1).sum(0) / n + std = ( + stds.view(n, -1).sum(0) / n + + ((means.view(n, -1) - mean) ** 2).view(n, -1).sum(0) / n + ) + return mean.detach(), std.detach() + + def aggregate_stats(self): + """ + Synchronize running_mean, and running_var. Call this before eval. + """ + if self.split_bn.track_running_stats: + ( + self.bn.running_mean.data, + self.bn.running_var.data, + ) = self._get_aggregated_mean_std( + self.split_bn.running_mean, + self.split_bn.running_var, + self.num_splits, + ) + + def forward(self, x): + if self.training: + n, c, t, h, w = x.shape + x = x.view(n // self.num_splits, c * self.num_splits, t, h, w) + x = self.split_bn(x) + x = x.view(n, c, t, h, w) + else: + x = self.bn(x) + if self.affine: + x = x * self.weight.view((-1, 1, 1, 1)) + x = x + self.bias.view((-1, 1, 1, 1)) + return x + + +class GroupGather(Function): + """ + GroupGather performs all gather on each of the local process/ GPU groups. + """ + + @staticmethod + def forward(ctx, input, num_sync_devices, num_groups): + """ + Perform forwarding, gathering the stats across different process/ GPU + group. + """ + ctx.num_sync_devices = num_sync_devices + ctx.num_groups = num_groups + + input_list = [ + torch.zeros_like(input) for k in range(du.get_local_size()) + ] + dist.all_gather( + input_list, input, async_op=False, group=du._LOCAL_PROCESS_GROUP + ) + + inputs = torch.stack(input_list, dim=0) + if num_groups > 1: + rank = du.get_local_rank() + group_idx = rank // num_sync_devices + inputs = inputs[ + group_idx + * num_sync_devices : (group_idx + 1) + * num_sync_devices + ] + inputs = torch.sum(inputs, dim=0) + return inputs + + @staticmethod + def backward(ctx, grad_output): + """ + Perform backwarding, gathering the gradients across different process/ GPU + group. + """ + grad_output_list = [ + torch.zeros_like(grad_output) for k in range(du.get_local_size()) + ] + dist.all_gather( + grad_output_list, + grad_output, + async_op=False, + group=du._LOCAL_PROCESS_GROUP, + ) + + grads = torch.stack(grad_output_list, dim=0) + if ctx.num_groups > 1: + rank = du.get_local_rank() + group_idx = rank // ctx.num_sync_devices + grads = grads[ + group_idx + * ctx.num_sync_devices : (group_idx + 1) + * ctx.num_sync_devices + ] + grads = torch.sum(grads, dim=0) + return grads, None, None + + +class NaiveSyncBatchNorm3d(nn.BatchNorm3d): + def __init__(self, num_sync_devices, **args): + """ + Naive version of Synchronized 3D BatchNorm. + Args: + num_sync_devices (int): number of device to sync. + args (list): other arguments. + """ + self.num_sync_devices = num_sync_devices + if self.num_sync_devices > 0: + assert du.get_local_size() % self.num_sync_devices == 0, ( + du.get_local_size(), + self.num_sync_devices, + ) + self.num_groups = du.get_local_size() // self.num_sync_devices + else: + self.num_sync_devices = du.get_local_size() + self.num_groups = 1 + super(NaiveSyncBatchNorm3d, self).__init__(**args) + + def forward(self, input): + if du.get_local_size() == 1 or not self.training: + return super().forward(input) + + assert input.shape[0] > 0, "SyncBatchNorm does not support empty inputs" + C = input.shape[1] + mean = torch.mean(input, dim=[0, 2, 3, 4]) + meansqr = torch.mean(input * input, dim=[0, 2, 3, 4]) + + vec = torch.cat([mean, meansqr], dim=0) + vec = GroupGather.apply(vec, self.num_sync_devices, self.num_groups) * ( + 1.0 / self.num_sync_devices + ) + + mean, meansqr = torch.split(vec, C) + var = meansqr - mean * mean + self.running_mean += self.momentum * (mean.detach() - self.running_mean) + self.running_var += self.momentum * (var.detach() - self.running_var) + + invstd = torch.rsqrt(var + self.eps) + scale = self.weight * invstd + bias = self.bias - mean * scale + scale = scale.reshape(1, -1, 1, 1, 1) + bias = bias.reshape(1, -1, 1, 1, 1) + return input * scale + bias diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/build.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/build.py" new file mode 100644 index 0000000..a88eb51 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/build.py" @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Model construction functions.""" + +import torch +from fvcore.common.registry import Registry + +MODEL_REGISTRY = Registry("MODEL") +MODEL_REGISTRY.__doc__ = """ +Registry for video model. + +The registered object will be called with `obj(cfg)`. +The call should return a `torch.nn.Module` object. +""" + + +def build_model(cfg, gpu_id=None): + """ + Builds the video model. + Args: + cfg (configs): configs that contains the hyper-parameters to build the + backbone. Details can be seen in slowfast/config/defaults.py. + gpu_id (Optional[int]): specify the gpu index to build model. + """ + if torch.cuda.is_available(): + assert ( + cfg.NUM_GPUS <= torch.cuda.device_count() + ), "Cannot use more GPU devices than available" + else: + assert ( + cfg.NUM_GPUS == 0 + ), "Cuda is not available. Please set `NUM_GPUS: 0 for running on CPUs." + + # Construct the model + name = cfg.MODEL.MODEL_NAME + model = MODEL_REGISTRY.get(name)(cfg) + + if cfg.NUM_GPUS: + if gpu_id is None: + # Determine the GPU used by the current process + cur_device = torch.cuda.current_device() + else: + cur_device = gpu_id + # Transfer the model to the current GPU device + model = model.cuda(device=cur_device) + # Use multi-process data parallel model in the multi-gpu setting + if cfg.NUM_GPUS > 1: + # Make model replica operate on the current device + model = torch.nn.parallel.DistributedDataParallel( + module=model, device_ids=[cur_device], output_device=cur_device + ) + return model diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/common.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/common.py" new file mode 100644 index 0000000..79fd494 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/common.py" @@ -0,0 +1,70 @@ +# Copyright (c) Facebook, Inc. and its affiliates. + +import torch +import torch.nn as nn + + +class Mlp(nn.Module): + def __init__( + self, + in_features, + hidden_features=None, + out_features=None, + act_layer=nn.GELU, + drop_rate=0.0, + ): + super().__init__() + self.drop_rate = drop_rate + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + if self.drop_rate > 0.0: + self.drop = nn.Dropout(drop_rate) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + if self.drop_rate > 0.0: + x = self.drop(x) + x = self.fc2(x) + if self.drop_rate > 0.0: + x = self.drop(x) + return x + + +class Permute(nn.Module): + def __init__(self, dims): + super().__init__() + self.dims = dims + + def forward(self, x): + return x.permute(*self.dims) + + +def drop_path(x, drop_prob: float = 0.0, training: bool = False): + """ + Stochastic Depth per sample. + """ + if drop_prob == 0.0 or not training: + return x + keep_prob = 1 - drop_prob + shape = (x.shape[0],) + (1,) * ( + x.ndim - 1 + ) # work with diff dim tensors, not just 2D ConvNets + mask = keep_prob + torch.rand(shape, dtype=x.dtype, device=x.device) + mask.floor_() # binarize + output = x.div(keep_prob) * mask + return output + + +class DropPath(nn.Module): + """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks).""" + + def __init__(self, drop_prob=None): + super(DropPath, self).__init__() + self.drop_prob = drop_prob + + def forward(self, x): + return drop_path(x, self.drop_prob, self.training) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/custom_video_model_builder.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/custom_video_model_builder.py" new file mode 100644 index 0000000..f261f67 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/custom_video_model_builder.py" @@ -0,0 +1,5 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + + +"""A More Flexible Video models.""" diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/head_helper.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/head_helper.py" new file mode 100644 index 0000000..faa40c9 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/head_helper.py" @@ -0,0 +1,408 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""ResNe(X)t Head helper.""" + +import torch +import torch.nn as nn +from detectron2.layers import ROIAlign + + +class ResNetRoIHead(nn.Module): + """ + ResNe(X)t RoI head. + """ + + def __init__( + self, + dim_in, + num_classes, + pool_size, + resolution, + scale_factor, + dropout_rate=0.0, + act_func="softmax", + aligned=True, + ): + """ + The `__init__` method of any subclass should also contain these + arguments. + ResNetRoIHead takes p pathways as input where p in [1, infty]. + + Args: + dim_in (list): the list of channel dimensions of the p inputs to the + ResNetHead. + num_classes (int): the channel dimensions of the p outputs to the + ResNetHead. + pool_size (list): the list of kernel sizes of p spatial temporal + poolings, temporal pool kernel size, spatial pool kernel size, + spatial pool kernel size in order. + resolution (list): the list of spatial output size from the ROIAlign. + scale_factor (list): the list of ratio to the input boxes by this + number. + dropout_rate (float): dropout rate. If equal to 0.0, perform no + dropout. + act_func (string): activation function to use. 'softmax': applies + softmax on the output. 'sigmoid': applies sigmoid on the output. + aligned (bool): if False, use the legacy implementation. If True, + align the results more perfectly. + Note: + Given a continuous coordinate c, its two neighboring pixel indices + (in our pixel model) are computed by floor (c - 0.5) and ceil + (c - 0.5). For example, c=1.3 has pixel neighbors with discrete + indices [0] and [1] (which are sampled from the underlying signal at + continuous coordinates 0.5 and 1.5). But the original roi_align + (aligned=False) does not subtract the 0.5 when computing neighboring + pixel indices and therefore it uses pixels with a slightly incorrect + alignment (relative to our pixel model) when performing bilinear + interpolation. + With `aligned=True`, we first appropriately scale the ROI and then + shift it by -0.5 prior to calling roi_align. This produces the + correct neighbors; It makes negligible differences to the model's + performance if ROIAlign is used together with conv layers. + """ + super(ResNetRoIHead, self).__init__() + assert ( + len({len(pool_size), len(dim_in)}) == 1 + ), "pathway dimensions are not consistent." + self.num_pathways = len(pool_size) + for pathway in range(self.num_pathways): + temporal_pool = nn.AvgPool3d( + [pool_size[pathway][0], 1, 1], stride=1 + ) + self.add_module("s{}_tpool".format(pathway), temporal_pool) + + roi_align = ROIAlign( + resolution[pathway], + spatial_scale=1.0 / scale_factor[pathway], + sampling_ratio=0, + aligned=aligned, + ) + self.add_module("s{}_roi".format(pathway), roi_align) + spatial_pool = nn.MaxPool2d(resolution[pathway], stride=1) + self.add_module("s{}_spool".format(pathway), spatial_pool) + + if dropout_rate > 0.0: + self.dropout = nn.Dropout(dropout_rate) + + # Perform FC in a fully convolutional manner. The FC layer will be + # initialized with a different std comparing to convolutional layers. + self.projection = nn.Linear(sum(dim_in), num_classes, bias=True) + + # Softmax for evaluation and testing. + if act_func == "softmax": + self.act = nn.Softmax(dim=1) + elif act_func == "sigmoid": + self.act = nn.Sigmoid() + else: + raise NotImplementedError( + "{} is not supported as an activation" + "function.".format(act_func) + ) + + def forward(self, inputs, bboxes): + assert ( + len(inputs) == self.num_pathways + ), "Input tensor does not contain {} pathway".format(self.num_pathways) + pool_out = [] + for pathway in range(self.num_pathways): + t_pool = getattr(self, "s{}_tpool".format(pathway)) + out = t_pool(inputs[pathway]) + assert out.shape[2] == 1 + out = torch.squeeze(out, 2) + + roi_align = getattr(self, "s{}_roi".format(pathway)) + out = roi_align(out, bboxes) + + s_pool = getattr(self, "s{}_spool".format(pathway)) + pool_out.append(s_pool(out)) + + # B C H W. + x = torch.cat(pool_out, 1) + + # Perform dropout. + if hasattr(self, "dropout"): + x = self.dropout(x) + + x = x.view(x.shape[0], -1) + x = self.projection(x) + x = self.act(x) + return x + + +class ResNetBasicHead(nn.Module): + """ + ResNe(X)t 3D head. + This layer performs a fully-connected projection during training, when the + input size is 1x1x1. It performs a convolutional projection during testing + when the input size is larger than 1x1x1. If the inputs are from multiple + different pathways, the inputs will be concatenated after pooling. + """ + + def __init__( + self, + dim_in, + num_classes, + pool_size, + dropout_rate=0.0, + act_func="softmax", + ): + """ + The `__init__` method of any subclass should also contain these + arguments. + ResNetBasicHead takes p pathways as input where p in [1, infty]. + + Args: + dim_in (list): the list of channel dimensions of the p inputs to the + ResNetHead. + num_classes (int): the channel dimensions of the p outputs to the + ResNetHead. + pool_size (list): the list of kernel sizes of p spatial temporal + poolings, temporal pool kernel size, spatial pool kernel size, + spatial pool kernel size in order. + dropout_rate (float): dropout rate. If equal to 0.0, perform no + dropout. + act_func (string): activation function to use. 'softmax': applies + softmax on the output. 'sigmoid': applies sigmoid on the output. + """ + super(ResNetBasicHead, self).__init__() + assert ( + len({len(pool_size), len(dim_in)}) == 1 + ), "pathway dimensions are not consistent." + self.num_pathways = len(pool_size) + + for pathway in range(self.num_pathways): + if pool_size[pathway] is None: + avg_pool = nn.AdaptiveAvgPool3d((1, 1, 1)) + else: + avg_pool = nn.AvgPool3d(pool_size[pathway], stride=1) + self.add_module("pathway{}_avgpool".format(pathway), avg_pool) + + if dropout_rate > 0.0: + self.dropout = nn.Dropout(dropout_rate) + # Perform FC in a fully convolutional manner. The FC layer will be + # initialized with a different std comparing to convolutional layers. + self.projection = nn.Linear(sum(dim_in), num_classes, bias=True) + + # Softmax for evaluation and testing. + if act_func == "softmax": + self.act = nn.Softmax(dim=4) + elif act_func == "sigmoid": + self.act = nn.Sigmoid() + else: + raise NotImplementedError( + "{} is not supported as an activation" + "function.".format(act_func) + ) + + def forward(self, inputs): + assert ( + len(inputs) == self.num_pathways + ), "Input tensor does not contain {} pathway".format(self.num_pathways) + pool_out = [] + for pathway in range(self.num_pathways): + m = getattr(self, "pathway{}_avgpool".format(pathway)) + pool_out.append(m(inputs[pathway])) + x = torch.cat(pool_out, 1) + # (N, C, T, H, W) -> (N, T, H, W, C). + x = x.permute((0, 2, 3, 4, 1)) + # Perform dropout. + if hasattr(self, "dropout"): + x = self.dropout(x) + x = self.projection(x) + + # Performs fully convlutional inference. + if not self.training: + x = self.act(x) + x = x.mean([1, 2, 3]) + + x = x.view(x.shape[0], -1) + return x + + +class X3DHead(nn.Module): + """ + X3D head. + This layer performs a fully-connected projection during training, when the + input size is 1x1x1. It performs a convolutional projection during testing + when the input size is larger than 1x1x1. If the inputs are from multiple + different pathways, the inputs will be concatenated after pooling. + """ + + def __init__( + self, + dim_in, + dim_inner, + dim_out, + num_classes, + pool_size, + dropout_rate=0.0, + act_func="softmax", + inplace_relu=True, + eps=1e-5, + bn_mmt=0.1, + norm_module=nn.BatchNorm3d, + bn_lin5_on=False, + ): + """ + The `__init__` method of any subclass should also contain these + arguments. + X3DHead takes a 5-dim feature tensor (BxCxTxHxW) as input. + + Args: + dim_in (float): the channel dimension C of the input. + num_classes (int): the channel dimensions of the output. + pool_size (float): a single entry list of kernel size for + spatiotemporal pooling for the TxHxW dimensions. + dropout_rate (float): dropout rate. If equal to 0.0, perform no + dropout. + act_func (string): activation function to use. 'softmax': applies + softmax on the output. 'sigmoid': applies sigmoid on the output. + inplace_relu (bool): if True, calculate the relu on the original + input without allocating new memory. + eps (float): epsilon for batch norm. + bn_mmt (float): momentum for batch norm. Noted that BN momentum in + PyTorch = 1 - BN momentum in Caffe2. + norm_module (nn.Module): nn.Module for the normalization layer. The + default is nn.BatchNorm3d. + bn_lin5_on (bool): if True, perform normalization on the features + before the classifier. + """ + super(X3DHead, self).__init__() + self.pool_size = pool_size + self.dropout_rate = dropout_rate + self.num_classes = num_classes + self.act_func = act_func + self.eps = eps + self.bn_mmt = bn_mmt + self.inplace_relu = inplace_relu + self.bn_lin5_on = bn_lin5_on + self._construct_head(dim_in, dim_inner, dim_out, norm_module) + + def _construct_head(self, dim_in, dim_inner, dim_out, norm_module): + + self.conv_5 = nn.Conv3d( + dim_in, + dim_inner, + kernel_size=(1, 1, 1), + stride=(1, 1, 1), + padding=(0, 0, 0), + bias=False, + ) + self.conv_5_bn = norm_module( + num_features=dim_inner, eps=self.eps, momentum=self.bn_mmt + ) + self.conv_5_relu = nn.ReLU(self.inplace_relu) + + if self.pool_size is None: + self.avg_pool = nn.AdaptiveAvgPool3d((1, 1, 1)) + else: + self.avg_pool = nn.AvgPool3d(self.pool_size, stride=1) + + self.lin_5 = nn.Conv3d( + dim_inner, + dim_out, + kernel_size=(1, 1, 1), + stride=(1, 1, 1), + padding=(0, 0, 0), + bias=False, + ) + if self.bn_lin5_on: + self.lin_5_bn = norm_module( + num_features=dim_out, eps=self.eps, momentum=self.bn_mmt + ) + self.lin_5_relu = nn.ReLU(self.inplace_relu) + + if self.dropout_rate > 0.0: + self.dropout = nn.Dropout(self.dropout_rate) + # Perform FC in a fully convolutional manner. The FC layer will be + # initialized with a different std comparing to convolutional layers. + self.projection = nn.Linear(dim_out, self.num_classes, bias=True) + + # Softmax for evaluation and testing. + if self.act_func == "softmax": + self.act = nn.Softmax(dim=4) + elif self.act_func == "sigmoid": + self.act = nn.Sigmoid() + else: + raise NotImplementedError( + "{} is not supported as an activation" + "function.".format(self.act_func) + ) + + def forward(self, inputs): + # In its current design the X3D head is only useable for a single + # pathway input. + assert len(inputs) == 1, "Input tensor does not contain 1 pathway" + x = self.conv_5(inputs[0]) + x = self.conv_5_bn(x) + x = self.conv_5_relu(x) + x = self.avg_pool(x) + + x = self.lin_5(x) + if self.bn_lin5_on: + x = self.lin_5_bn(x) + x = self.lin_5_relu(x) + + # (N, C, T, H, W) -> (N, T, H, W, C). + x = x.permute((0, 2, 3, 4, 1)) + # Perform dropout. + if hasattr(self, "dropout"): + x = self.dropout(x) + x = self.projection(x) + + # Performs fully convlutional inference. + if not self.training: + x = self.act(x) + x = x.mean([1, 2, 3]) + + x = x.view(x.shape[0], -1) + return x + + +class TransformerBasicHead(nn.Module): + """ + BasicHead. No pool. + """ + + def __init__( + self, + dim_in, + num_classes, + dropout_rate=0.0, + act_func="softmax", + ): + """ + Perform linear projection and activation as head for tranformers. + Args: + dim_in (int): the channel dimension of the input to the head. + num_classes (int): the channel dimensions of the output to the head. + dropout_rate (float): dropout rate. If equal to 0.0, perform no + dropout. + act_func (string): activation function to use. 'softmax': applies + softmax on the output. 'sigmoid': applies sigmoid on the output. + """ + super(TransformerBasicHead, self).__init__() + if dropout_rate > 0.0: + self.dropout = nn.Dropout(dropout_rate) + self.projection = nn.Linear(dim_in, num_classes, bias=True) + + # Softmax for evaluation and testing. + if act_func == "softmax": + self.act = nn.Softmax(dim=1) + elif act_func == "sigmoid": + self.act = nn.Sigmoid() + else: + raise NotImplementedError( + "{} is not supported as an activation" + "function.".format(act_func) + ) + + def forward(self, x): + if hasattr(self, "dropout"): + x = self.dropout(x) + x = self.projection(x) + + if not self.training: + x = self.act(x) + return x diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/losses.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/losses.py" new file mode 100644 index 0000000..55c71ce --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/losses.py" @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Loss functions.""" + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class SoftTargetCrossEntropy(nn.Module): + """ + Cross entropy loss with soft target. + """ + + def __init__(self, reduction="mean"): + """ + Args: + reduction (str): specifies reduction to apply to the output. It can be + "mean" (default) or "none". + """ + super(SoftTargetCrossEntropy, self).__init__() + self.reduction = reduction + + def forward(self, x, y): + loss = torch.sum(-y * F.log_softmax(x, dim=-1), dim=-1) + if self.reduction == "mean": + return loss.mean() + elif self.reduction == "none": + return loss + else: + raise NotImplementedError + + +_LOSSES = { + "cross_entropy": nn.CrossEntropyLoss, + "bce": nn.BCELoss, + "bce_logit": nn.BCEWithLogitsLoss, + "soft_cross_entropy": SoftTargetCrossEntropy, +} + + +def get_loss_func(loss_name): + """ + Retrieve the loss given the loss name. + Args (int): + loss_name: the name of the loss to use. + """ + if loss_name not in _LOSSES.keys(): + raise NotImplementedError("Loss {} is not supported".format(loss_name)) + return _LOSSES[loss_name] diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/nonlocal_helper.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/nonlocal_helper.py" new file mode 100644 index 0000000..6e68d05 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/nonlocal_helper.py" @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Non-local helper""" + +import torch +import torch.nn as nn + + +class Nonlocal(nn.Module): + """ + Builds Non-local Neural Networks as a generic family of building + blocks for capturing long-range dependencies. Non-local Network + computes the response at a position as a weighted sum of the + features at all positions. This building block can be plugged into + many computer vision architectures. + More details in the paper: https://arxiv.org/pdf/1711.07971.pdf + """ + + def __init__( + self, + dim, + dim_inner, + pool_size=None, + instantiation="softmax", + zero_init_final_conv=False, + zero_init_final_norm=True, + norm_eps=1e-5, + norm_momentum=0.1, + norm_module=nn.BatchNorm3d, + ): + """ + Args: + dim (int): number of dimension for the input. + dim_inner (int): number of dimension inside of the Non-local block. + pool_size (list): the kernel size of spatial temporal pooling, + temporal pool kernel size, spatial pool kernel size, spatial + pool kernel size in order. By default pool_size is None, + then there would be no pooling used. + instantiation (string): supports two different instantiation method: + "dot_product": normalizing correlation matrix with L2. + "softmax": normalizing correlation matrix with Softmax. + zero_init_final_conv (bool): If true, zero initializing the final + convolution of the Non-local block. + zero_init_final_norm (bool): + If true, zero initializing the final batch norm of the Non-local + block. + norm_module (nn.Module): nn.Module for the normalization layer. The + default is nn.BatchNorm3d. + """ + super(Nonlocal, self).__init__() + self.dim = dim + self.dim_inner = dim_inner + self.pool_size = pool_size + self.instantiation = instantiation + self.use_pool = ( + False + if pool_size is None + else any((size > 1 for size in pool_size)) + ) + self.norm_eps = norm_eps + self.norm_momentum = norm_momentum + self._construct_nonlocal( + zero_init_final_conv, zero_init_final_norm, norm_module + ) + + def _construct_nonlocal( + self, zero_init_final_conv, zero_init_final_norm, norm_module + ): + # Three convolution heads: theta, phi, and g. + self.conv_theta = nn.Conv3d( + self.dim, self.dim_inner, kernel_size=1, stride=1, padding=0 + ) + self.conv_phi = nn.Conv3d( + self.dim, self.dim_inner, kernel_size=1, stride=1, padding=0 + ) + self.conv_g = nn.Conv3d( + self.dim, self.dim_inner, kernel_size=1, stride=1, padding=0 + ) + + # Final convolution output. + self.conv_out = nn.Conv3d( + self.dim_inner, self.dim, kernel_size=1, stride=1, padding=0 + ) + # Zero initializing the final convolution output. + self.conv_out.zero_init = zero_init_final_conv + + # TODO: change the name to `norm` + self.bn = norm_module( + num_features=self.dim, + eps=self.norm_eps, + momentum=self.norm_momentum, + ) + # Zero initializing the final bn. + self.bn.transform_final_bn = zero_init_final_norm + + # Optional to add the spatial-temporal pooling. + if self.use_pool: + self.pool = nn.MaxPool3d( + kernel_size=self.pool_size, + stride=self.pool_size, + padding=[0, 0, 0], + ) + + def forward(self, x): + x_identity = x + N, C, T, H, W = x.size() + + theta = self.conv_theta(x) + + # Perform temporal-spatial pooling to reduce the computation. + if self.use_pool: + x = self.pool(x) + + phi = self.conv_phi(x) + g = self.conv_g(x) + + theta = theta.view(N, self.dim_inner, -1) + phi = phi.view(N, self.dim_inner, -1) + g = g.view(N, self.dim_inner, -1) + + # (N, C, TxHxW) * (N, C, TxHxW) => (N, TxHxW, TxHxW). + theta_phi = torch.einsum("nct,ncp->ntp", (theta, phi)) + # For original Non-local paper, there are two main ways to normalize + # the affinity tensor: + # 1) Softmax normalization (norm on exp). + # 2) dot_product normalization. + if self.instantiation == "softmax": + # Normalizing the affinity tensor theta_phi before softmax. + theta_phi = theta_phi * (self.dim_inner ** -0.5) + theta_phi = nn.functional.softmax(theta_phi, dim=2) + elif self.instantiation == "dot_product": + spatial_temporal_dim = theta_phi.shape[2] + theta_phi = theta_phi / spatial_temporal_dim + else: + raise NotImplementedError( + "Unknown norm type {}".format(self.instantiation) + ) + + # (N, TxHxW, TxHxW) * (N, C, TxHxW) => (N, C, TxHxW). + theta_phi_g = torch.einsum("ntg,ncg->nct", (theta_phi, g)) + + # (N, C, TxHxW) => (N, C, T, H, W). + theta_phi_g = theta_phi_g.view(N, self.dim_inner, T, H, W) + + p = self.conv_out(theta_phi_g) + p = self.bn(p) + return x_identity + p diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/operators.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/operators.py" new file mode 100644 index 0000000..0825669 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/operators.py" @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Custom operators.""" + +import torch +import torch.nn as nn + + +class Swish(nn.Module): + """Swish activation function: x * sigmoid(x).""" + + def __init__(self): + super(Swish, self).__init__() + + def forward(self, x): + return SwishEfficient.apply(x) + + +class SwishEfficient(torch.autograd.Function): + """Swish activation function: x * sigmoid(x).""" + + @staticmethod + def forward(ctx, x): + result = x * torch.sigmoid(x) + ctx.save_for_backward(x) + return result + + @staticmethod + def backward(ctx, grad_output): + x = ctx.saved_variables[0] + sigmoid_x = torch.sigmoid(x) + return grad_output * (sigmoid_x * (1 + x * (1 - sigmoid_x))) + + +class SE(nn.Module): + """Squeeze-and-Excitation (SE) block w/ Swish: AvgPool, FC, Swish, FC, Sigmoid.""" + + def _round_width(self, width, multiplier, min_width=8, divisor=8): + """ + Round width of filters based on width multiplier + Args: + width (int): the channel dimensions of the input. + multiplier (float): the multiplication factor. + min_width (int): the minimum width after multiplication. + divisor (int): the new width should be dividable by divisor. + """ + if not multiplier: + return width + + width *= multiplier + min_width = min_width or divisor + width_out = max( + min_width, int(width + divisor / 2) // divisor * divisor + ) + if width_out < 0.9 * width: + width_out += divisor + return int(width_out) + + def __init__(self, dim_in, ratio, relu_act=True): + """ + Args: + dim_in (int): the channel dimensions of the input. + ratio (float): the channel reduction ratio for squeeze. + relu_act (bool): whether to use ReLU activation instead + of Swish (default). + divisor (int): the new width should be dividable by divisor. + """ + super(SE, self).__init__() + self.avg_pool = nn.AdaptiveAvgPool3d((1, 1, 1)) + dim_fc = self._round_width(dim_in, ratio) + self.fc1 = nn.Conv3d(dim_in, dim_fc, 1, bias=True) + self.fc1_act = nn.ReLU() if relu_act else Swish() + self.fc2 = nn.Conv3d(dim_fc, dim_in, 1, bias=True) + + self.fc2_sig = nn.Sigmoid() + + def forward(self, x): + x_in = x + for module in self.children(): + x = module(x) + return x_in * x diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/optimizer.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/optimizer.py" new file mode 100644 index 0000000..3a14392 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/optimizer.py" @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Optimizer.""" + +import torch + +import slowfast.utils.lr_policy as lr_policy + + +def construct_optimizer(model, cfg): + """ + Construct a stochastic gradient descent or ADAM optimizer with momentum. + Details can be found in: + Herbert Robbins, and Sutton Monro. "A stochastic approximation method." + and + Diederik P.Kingma, and Jimmy Ba. + "Adam: A Method for Stochastic Optimization." + + Args: + model (model): model to perform stochastic gradient descent + optimization or ADAM optimization. + cfg (config): configs of hyper-parameters of SGD or ADAM, includes base + learning rate, momentum, weight_decay, dampening, and etc. + """ + bn_parameters = [] + non_bn_parameters = [] + zero_parameters = [] + skip = {} + if hasattr(model, "no_weight_decay"): + skip = model.no_weight_decay() + + for name, m in model.named_modules(): + is_bn = isinstance(m, torch.nn.modules.batchnorm._NormBase) + for p in m.parameters(recurse=False): + if not p.requires_grad: + continue + if is_bn: + bn_parameters.append(p) + elif name in skip or ( + (len(p.shape) == 1 or name.endswith(".bias")) + and cfg.SOLVER.ZERO_WD_1D_PARAM + ): + zero_parameters.append(p) + else: + non_bn_parameters.append(p) + + optim_params = [ + {"params": bn_parameters, "weight_decay": cfg.BN.WEIGHT_DECAY}, + {"params": non_bn_parameters, "weight_decay": cfg.SOLVER.WEIGHT_DECAY}, + {"params": zero_parameters, "weight_decay": 0.0}, + ] + optim_params = [x for x in optim_params if len(x["params"])] + + # Check all parameters will be passed into optimizer. + assert len(list(model.parameters())) == len(non_bn_parameters) + len( + bn_parameters + ) + len( + zero_parameters + ), "parameter size does not match: {} + {} + {} != {}".format( + len(non_bn_parameters), + len(bn_parameters), + len(zero_parameters), + len(list(model.parameters())), + ) + print( + "bn {}, non bn {}, zero {}".format( + len(bn_parameters), len(non_bn_parameters), len(zero_parameters) + ) + ) + + if cfg.SOLVER.OPTIMIZING_METHOD == "sgd": + return torch.optim.SGD( + optim_params, + lr=cfg.SOLVER.BASE_LR, + momentum=cfg.SOLVER.MOMENTUM, + weight_decay=cfg.SOLVER.WEIGHT_DECAY, + dampening=cfg.SOLVER.DAMPENING, + nesterov=cfg.SOLVER.NESTEROV, + ) + elif cfg.SOLVER.OPTIMIZING_METHOD == "adam": + return torch.optim.Adam( + optim_params, + lr=cfg.SOLVER.BASE_LR, + betas=(0.9, 0.999), + weight_decay=cfg.SOLVER.WEIGHT_DECAY, + ) + elif cfg.SOLVER.OPTIMIZING_METHOD == "adamw": + return torch.optim.AdamW( + optim_params, + lr=cfg.SOLVER.BASE_LR, + eps=1e-08, + weight_decay=cfg.SOLVER.WEIGHT_DECAY, + ) + else: + raise NotImplementedError( + "Does not support {} optimizer".format(cfg.SOLVER.OPTIMIZING_METHOD) + ) + + +def get_epoch_lr(cur_epoch, cfg): + """ + Retrieves the lr for the given epoch (as specified by the lr policy). + Args: + cfg (config): configs of hyper-parameters of ADAM, includes base + learning rate, betas, and weight decay. + cur_epoch (float): the number of epoch of the current training stage. + """ + return lr_policy.get_lr_at_epoch(cfg, cur_epoch) + + +def set_lr(optimizer, new_lr): + """ + Sets the optimizer lr to the specified value. + Args: + optimizer (optim): the optimizer using to optimize the current network. + new_lr (float): the new learning rate to set. + """ + for param_group in optimizer.param_groups: + param_group["lr"] = new_lr diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/ptv_model_builder.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/ptv_model_builder.py" new file mode 100644 index 0000000..09052a2 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/ptv_model_builder.py" @@ -0,0 +1,701 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + + +"""Video models using PyTorchVideo model builder.""" + +from functools import partial +import torch.nn as nn +from detectron2.layers import ROIAlign + +from slowfast.models.batchnorm_helper import get_norm +from slowfast.models.video_model_builder import _POOL1, _TEMPORAL_KERNEL_BASIS + +from pytorchvideo.models.csn import create_csn +from pytorchvideo.models.head import ( + create_res_basic_head, + create_res_roi_pooling_head, +) +from pytorchvideo.models.r2plus1d import ( + create_2plus1d_bottleneck_block, + create_r2plus1d, +) +from pytorchvideo.models.resnet import create_bottleneck_block, create_resnet +from pytorchvideo.models.slowfast import create_slowfast +from pytorchvideo.models.x3d import ( + Swish, + create_x3d, + create_x3d_bottleneck_block, +) + +from .build import MODEL_REGISTRY + + +def get_head_act(act_func): + """ + Return the actual head activation function given the activation fucntion name. + + Args: + act_func (string): activation function to use. 'softmax': applies + softmax on the output. 'sigmoid': applies sigmoid on the output. + Returns: + nn.Module: the activation layer. + """ + if act_func == "softmax": + return nn.Softmax(dim=1) + elif act_func == "sigmoid": + return nn.Sigmoid() + else: + raise NotImplementedError( + "{} is not supported as a head activation " + "function.".format(act_func) + ) + + +@MODEL_REGISTRY.register() +class PTVResNet(nn.Module): + """ + ResNet models using PyTorchVideo model builder. + """ + + def __init__(self, cfg): + """ + The `__init__` method of any subclass should also contain these + arguments. + + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + super(PTVResNet, self).__init__() + + assert ( + cfg.RESNET.STRIDE_1X1 is False + ), "STRIDE_1x1 must be True for PTVResNet" + assert ( + cfg.RESNET.TRANS_FUNC == "bottleneck_transform" + ), f"Unsupported TRANS_FUNC type {cfg.RESNET.TRANS_FUNC} for PTVResNet" + assert cfg.MODEL.ARCH in [ + "c2d", + "slow", + "i3d", + ], f"Unsupported MODEL.ARCH type {cfg.MODEL.ARCH} for PTVResNet" + + self.detection_mode = cfg.DETECTION.ENABLE + self._construct_network(cfg) + + def _construct_network(self, cfg): + """ + Builds a single pathway ResNet model. + + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + + # Params from configs. + norm_module = get_norm(cfg) + head_act = get_head_act(cfg.MODEL.HEAD_ACT) + pool_size = _POOL1[cfg.MODEL.ARCH] + num_groups = cfg.RESNET.NUM_GROUPS + spatial_dilations = cfg.RESNET.SPATIAL_DILATIONS + spatial_strides = cfg.RESNET.SPATIAL_STRIDES + temp_kernel = _TEMPORAL_KERNEL_BASIS[cfg.MODEL.ARCH] + stage1_pool = pool_size[0][0] != 1 or len(set(pool_size[0])) > 1 + stage_spatial_stride = ( + spatial_strides[0][0], + spatial_strides[1][0], + spatial_strides[2][0], + spatial_strides[3][0], + ) + if cfg.MODEL.ARCH == "i3d": + stage_conv_a_kernel_size = ( + (3, 1, 1), + [(3, 1, 1), (1, 1, 1)], + [(3, 1, 1), (1, 1, 1)], + [(1, 1, 1), (3, 1, 1)], + ) + else: + stage_conv_a_kernel_size = ( + (temp_kernel[1][0][0], 1, 1), + (temp_kernel[2][0][0], 1, 1), + (temp_kernel[3][0][0], 1, 1), + (temp_kernel[4][0][0], 1, 1), + ) + + # Head from config + if cfg.DETECTION.ENABLE: + self.detection_head = create_res_roi_pooling_head( + in_features=cfg.RESNET.WIDTH_PER_GROUP * 2 ** (4 + 1), + out_features=cfg.MODEL.NUM_CLASSES, + pool=nn.AvgPool3d, + output_size=(1, 1, 1), + pool_kernel_size=( + cfg.DATA.NUM_FRAMES // pool_size[0][0], + 1, + 1, + ), + dropout_rate=cfg.MODEL.DROPOUT_RATE, + activation=None, + output_with_global_average=False, + pool_spatial=nn.MaxPool2d, + resolution=[cfg.DETECTION.ROI_XFORM_RESOLUTION] * 2, + spatial_scale=1.0 / float(cfg.DETECTION.SPATIAL_SCALE_FACTOR), + sampling_ratio=0, + roi=ROIAlign, + ) + + self.model = create_resnet( + # Input clip configs. + input_channel=cfg.DATA.INPUT_CHANNEL_NUM[0], + # Model configs. + model_depth=cfg.RESNET.DEPTH, + model_num_class=cfg.MODEL.NUM_CLASSES, + dropout_rate=cfg.MODEL.DROPOUT_RATE, + # Normalization configs. + norm=norm_module, + # Activation configs. + activation=partial(nn.ReLU, inplace=cfg.RESNET.INPLACE_RELU), + # Stem configs. + stem_dim_out=cfg.RESNET.WIDTH_PER_GROUP, + stem_conv_kernel_size=(temp_kernel[0][0][0], 7, 7), + stem_conv_stride=(1, 2, 2), + stem_pool=nn.MaxPool3d, + stem_pool_kernel_size=(1, 3, 3), + stem_pool_stride=(1, 2, 2), + # Stage configs. + stage1_pool=nn.MaxPool3d if stage1_pool else None, + stage1_pool_kernel_size=pool_size[0], + stage_conv_a_kernel_size=stage_conv_a_kernel_size, + stage_conv_b_kernel_size=( + (1, 3, 3), + (1, 3, 3), + (1, 3, 3), + (1, 3, 3), + ), + stage_conv_b_num_groups=( + num_groups, + num_groups, + num_groups, + num_groups, + ), + stage_conv_b_dilation=( + (1, spatial_dilations[0][0], spatial_dilations[0][0]), + (1, spatial_dilations[1][0], spatial_dilations[1][0]), + (1, spatial_dilations[2][0], spatial_dilations[2][0]), + (1, spatial_dilations[3][0], spatial_dilations[3][0]), + ), + stage_spatial_h_stride=stage_spatial_stride, + stage_spatial_w_stride=stage_spatial_stride, + stage_temporal_stride=(1, 1, 1, 1), + bottleneck=create_bottleneck_block, + # Head configs. + head=create_res_basic_head if not self.detection_mode else None, + head_pool=nn.AvgPool3d, + head_pool_kernel_size=( + cfg.DATA.NUM_FRAMES // pool_size[0][0], + cfg.DATA.TRAIN_CROP_SIZE // 32 // pool_size[0][1], + cfg.DATA.TRAIN_CROP_SIZE // 32 // pool_size[0][2], + ), + head_activation=None, + head_output_with_global_average=False, + ) + + self.post_act = head_act + + def forward(self, x, bboxes=None): + x = x[0] + x = self.model(x) + if self.detection_mode: + x = self.detection_head(x, bboxes) + x = self.post_act(x) + else: + # Performs fully convlutional inference. + if not self.training: + x = self.post_act(x) + x = x.mean([2, 3, 4]) + x = x.view(x.shape[0], -1) + return x + + +@MODEL_REGISTRY.register() +class PTVSlowFast(nn.Module): + def __init__(self, cfg): + """ + The `__init__` method of any subclass should also contain these + arguments. + + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + super(PTVSlowFast, self).__init__() + + assert ( + cfg.RESNET.STRIDE_1X1 is False + ), "STRIDE_1x1 must be True for PTVSlowFast" + assert ( + cfg.RESNET.TRANS_FUNC == "bottleneck_transform" + ), f"Unsupported TRANS_FUNC type {cfg.RESNET.TRANS_FUNC} for PTVSlowFast" + + self.detection_mode = cfg.DETECTION.ENABLE + self._construct_network(cfg) + + def _construct_network(self, cfg): + """ + Builds a SlowFast model. + + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + _MODEL_STAGE_DEPTH = {50: (3, 4, 6, 3), 101: (3, 4, 23, 3)} + + # Params from configs. + norm_module = get_norm(cfg) + pool_size = _POOL1[cfg.MODEL.ARCH] + num_groups = cfg.RESNET.NUM_GROUPS + width_per_group = cfg.RESNET.WIDTH_PER_GROUP + spatial_dilations = cfg.RESNET.SPATIAL_DILATIONS + spatial_strides = cfg.RESNET.SPATIAL_STRIDES + temp_kernel = _TEMPORAL_KERNEL_BASIS[cfg.MODEL.ARCH] + num_block_temp_kernel = cfg.RESNET.NUM_BLOCK_TEMP_KERNEL + stage_depth = _MODEL_STAGE_DEPTH[cfg.RESNET.DEPTH] + + stage_conv_a_kernel_sizes = [[], []] + for pathway in range(2): + for stage in range(4): + stage_conv_a_kernel_sizes[pathway].append( + ((temp_kernel[stage + 1][pathway][0], 1, 1),) + * num_block_temp_kernel[stage][pathway] + + ((1, 1, 1),) + * ( + stage_depth[stage] + - num_block_temp_kernel[stage][pathway] + ) + ) + + # Head from config + # Number of stages = 4 + stage_dim_in = cfg.RESNET.WIDTH_PER_GROUP * 2 ** (4 + 1) + head_in_features = stage_dim_in + for reduction_ratio in cfg.SLOWFAST.BETA_INV: + head_in_features = ( + head_in_features + stage_dim_in // reduction_ratio + ) + + if cfg.DETECTION.ENABLE: + self.detection_head = create_res_roi_pooling_head( + in_features=head_in_features, + out_features=cfg.MODEL.NUM_CLASSES, + pool=None, + output_size=(1, 1, 1), + dropout_rate=cfg.MODEL.DROPOUT_RATE, + activation=None, + output_with_global_average=False, + pool_spatial=nn.MaxPool2d, + resolution=[cfg.DETECTION.ROI_XFORM_RESOLUTION] * 2, + spatial_scale=1.0 / float(cfg.DETECTION.SPATIAL_SCALE_FACTOR), + sampling_ratio=0, + roi=ROIAlign, + ) + head_pool_kernel_sizes = ( + ( + cfg.DATA.NUM_FRAMES + // cfg.SLOWFAST.ALPHA + // pool_size[0][0], + 1, + 1, + ), + (cfg.DATA.NUM_FRAMES // pool_size[1][0], 1, 1), + ) + else: + head_pool_kernel_sizes = ( + ( + cfg.DATA.NUM_FRAMES + // cfg.SLOWFAST.ALPHA + // pool_size[0][0], + cfg.DATA.TRAIN_CROP_SIZE // 32 // pool_size[0][1], + cfg.DATA.TRAIN_CROP_SIZE // 32 // pool_size[0][2], + ), + ( + cfg.DATA.NUM_FRAMES // pool_size[1][0], + cfg.DATA.TRAIN_CROP_SIZE // 32 // pool_size[1][1], + cfg.DATA.TRAIN_CROP_SIZE // 32 // pool_size[1][2], + ), + ) + + self.model = create_slowfast( + # SlowFast configs. + slowfast_channel_reduction_ratio=cfg.SLOWFAST.BETA_INV, + slowfast_conv_channel_fusion_ratio=cfg.SLOWFAST.FUSION_CONV_CHANNEL_RATIO, + slowfast_fusion_conv_kernel_size=( + cfg.SLOWFAST.FUSION_KERNEL_SZ, + 1, + 1, + ), + slowfast_fusion_conv_stride=(cfg.SLOWFAST.ALPHA, 1, 1), + # Input clip configs. + input_channels=cfg.DATA.INPUT_CHANNEL_NUM, + # Model configs. + model_depth=cfg.RESNET.DEPTH, + model_num_class=cfg.MODEL.NUM_CLASSES, + dropout_rate=cfg.MODEL.DROPOUT_RATE, + # Normalization configs. + norm=norm_module, + # Activation configs. + activation=partial(nn.ReLU, inplace=cfg.RESNET.INPLACE_RELU), + # Stem configs. + stem_dim_outs=( + width_per_group, + width_per_group // cfg.SLOWFAST.BETA_INV, + ), + stem_conv_kernel_sizes=( + (temp_kernel[0][0][0], 7, 7), + (temp_kernel[0][1][0], 7, 7), + ), + stem_conv_strides=((1, 2, 2), (1, 2, 2)), + stem_pool=nn.MaxPool3d, + stem_pool_kernel_sizes=((1, 3, 3), (1, 3, 3)), + stem_pool_strides=((1, 2, 2), (1, 2, 2)), + # Stage configs. + stage_conv_a_kernel_sizes=stage_conv_a_kernel_sizes, + stage_conv_b_kernel_sizes=( + ((1, 3, 3), (1, 3, 3), (1, 3, 3), (1, 3, 3)), + ((1, 3, 3), (1, 3, 3), (1, 3, 3), (1, 3, 3)), + ), + stage_conv_b_num_groups=( + (num_groups, num_groups, num_groups, num_groups), + (num_groups, num_groups, num_groups, num_groups), + ), + stage_conv_b_dilations=( + ( + (1, spatial_dilations[0][0], spatial_dilations[0][0]), + (1, spatial_dilations[1][0], spatial_dilations[1][0]), + (1, spatial_dilations[2][0], spatial_dilations[2][0]), + (1, spatial_dilations[3][0], spatial_dilations[3][0]), + ), + ( + (1, spatial_dilations[0][1], spatial_dilations[0][1]), + (1, spatial_dilations[1][1], spatial_dilations[1][1]), + (1, spatial_dilations[1][1], spatial_dilations[1][1]), + (1, spatial_dilations[1][1], spatial_dilations[1][1]), + ), + ), + stage_spatial_strides=( + ( + spatial_strides[0][0], + spatial_strides[1][0], + spatial_strides[2][0], + spatial_strides[3][0], + ), + ( + spatial_strides[0][1], + spatial_strides[1][1], + spatial_strides[2][1], + spatial_strides[3][1], + ), + ), + stage_temporal_strides=((1, 1, 1, 1), (1, 1, 1, 1)), + bottleneck=create_bottleneck_block, + # Head configs. + head=create_res_basic_head if not self.detection_mode else None, + head_pool=nn.AvgPool3d, + head_pool_kernel_sizes=head_pool_kernel_sizes, + head_activation=None, + head_output_with_global_average=False, + ) + + self.post_act = get_head_act(cfg.MODEL.HEAD_ACT) + + def forward(self, x, bboxes=None): + x = self.model(x) + if self.detection_mode: + x = self.detection_head(x, bboxes) + x = self.post_act(x) + else: + # Performs fully convlutional inference. + if not self.training: + x = self.post_act(x) + x = x.mean([2, 3, 4]) + x = x.view(x.shape[0], -1) + return x + + +@MODEL_REGISTRY.register() +class PTVX3D(nn.Module): + def __init__(self, cfg): + """ + The `__init__` method of any subclass should also contain these + arguments. + + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + super(PTVX3D, self).__init__() + + assert ( + cfg.RESNET.STRIDE_1X1 is False + ), "STRIDE_1x1 must be True for PTVX3D" + assert ( + cfg.RESNET.TRANS_FUNC == "x3d_transform" + ), f"Unsupported TRANS_FUNC type {cfg.RESNET.TRANS_FUNC} for PTVX3D" + assert ( + cfg.DETECTION.ENABLE is False + ), "Detection model is not supported for PTVX3D yet." + + self._construct_network(cfg) + + def _construct_network(self, cfg): + """ + Builds a X3D model. + + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + + # Params from configs. + norm_module = get_norm(cfg) + temp_kernel = _TEMPORAL_KERNEL_BASIS[cfg.MODEL.ARCH] + + self.model = create_x3d( + # Input clip configs. + input_channel=cfg.DATA.INPUT_CHANNEL_NUM[0], + input_clip_length=cfg.DATA.NUM_FRAMES, + input_crop_size=cfg.DATA.TRAIN_CROP_SIZE, + # Model configs. + model_num_class=cfg.MODEL.NUM_CLASSES, + dropout_rate=cfg.MODEL.DROPOUT_RATE, + width_factor=cfg.X3D.WIDTH_FACTOR, + depth_factor=cfg.X3D.DEPTH_FACTOR, + # Normalization configs. + norm=norm_module, + norm_eps=1e-5, + norm_momentum=0.1, + # Activation configs. + activation=partial(nn.ReLU, inplace=cfg.RESNET.INPLACE_RELU), + # Stem configs. + stem_dim_in=cfg.X3D.DIM_C1, + stem_conv_kernel_size=(temp_kernel[0][0][0], 3, 3), + stem_conv_stride=(1, 2, 2), + # Stage configs. + stage_conv_kernel_size=( + (temp_kernel[1][0][0], 3, 3), + (temp_kernel[2][0][0], 3, 3), + (temp_kernel[3][0][0], 3, 3), + (temp_kernel[4][0][0], 3, 3), + ), + stage_spatial_stride=(2, 2, 2, 2), + stage_temporal_stride=(1, 1, 1, 1), + bottleneck=create_x3d_bottleneck_block, + bottleneck_factor=cfg.X3D.BOTTLENECK_FACTOR, + se_ratio=0.0625, + inner_act=Swish, + # Head configs. + head_dim_out=cfg.X3D.DIM_C5, + head_pool_act=partial(nn.ReLU, inplace=cfg.RESNET.INPLACE_RELU), + head_bn_lin5_on=cfg.X3D.BN_LIN5, + head_activation=None, + head_output_with_global_average=False, + ) + + self.post_act = get_head_act(cfg.MODEL.HEAD_ACT) + + def forward(self, x, bboxes=None): + x = x[0] + x = self.model(x) + # Performs fully convlutional inference. + if not self.training: + x = self.post_act(x) + x = x.mean([2, 3, 4]) + + x = x.reshape(x.shape[0], -1) + return x + + +@MODEL_REGISTRY.register() +class PTVCSN(nn.Module): + """ + CSN models using PyTorchVideo model builder. + """ + + def __init__(self, cfg): + """ + The `__init__` method of any subclass should also contain these + arguments. + + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + super(PTVCSN, self).__init__() + + assert ( + cfg.DETECTION.ENABLE is False + ), "Detection model is not supported for PTVCSN yet." + + self._construct_network(cfg) + + def _construct_network(self, cfg): + """ + Builds a single pathway ResNet model. + + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + + # Params from configs. + norm_module = get_norm(cfg) + + self.model = create_csn( + # Input clip configs. + input_channel=cfg.DATA.INPUT_CHANNEL_NUM[0], + # Model configs. + model_depth=cfg.RESNET.DEPTH, + model_num_class=cfg.MODEL.NUM_CLASSES, + dropout_rate=cfg.MODEL.DROPOUT_RATE, + # Normalization configs. + norm=norm_module, + # Activation configs. + activation=partial(nn.ReLU, inplace=cfg.RESNET.INPLACE_RELU), + # Stem configs. + stem_dim_out=cfg.RESNET.WIDTH_PER_GROUP, + stem_conv_kernel_size=(3, 7, 7), + stem_conv_stride=(1, 2, 2), + stem_pool=nn.MaxPool3d, + stem_pool_kernel_size=(1, 3, 3), + stem_pool_stride=(1, 2, 2), + # Stage configs. + stage_conv_a_kernel_size=(1, 1, 1), + stage_conv_b_kernel_size=(3, 3, 3), + stage_conv_b_width_per_group=1, + stage_spatial_stride=(1, 2, 2, 2), + stage_temporal_stride=(1, 2, 2, 2), + bottleneck=create_bottleneck_block, + # Head configs. + head_pool=nn.AvgPool3d, + head_pool_kernel_size=( + cfg.DATA.NUM_FRAMES // 8, + cfg.DATA.TRAIN_CROP_SIZE // 32, + cfg.DATA.TRAIN_CROP_SIZE // 32, + ), + head_activation=None, + head_output_with_global_average=False, + ) + + self.post_act = get_head_act(cfg.MODEL.HEAD_ACT) + + def forward(self, x, bboxes=None): + x = x[0] + x = self.model(x) + # Performs fully convlutional inference. + if not self.training: + x = self.post_act(x) + x = x.mean([2, 3, 4]) + + x = x.reshape(x.shape[0], -1) + return x + + +@MODEL_REGISTRY.register() +class PTVR2plus1D(nn.Module): + """ + R(2+1)D models using PyTorchVideo model builder. + """ + + def __init__(self, cfg): + """ + The `__init__` method of any subclass should also contain these + arguments. + + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + super(PTVR2plus1D, self).__init__() + + assert ( + cfg.DETECTION.ENABLE is False + ), "Detection model is not supported for PTVR2plus1D yet." + + self._construct_network(cfg) + + def _construct_network(self, cfg): + """ + Builds a single pathway R(2+1)D model. + + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + self.model = create_r2plus1d( + # Input clip configs. + input_channel=cfg.DATA.INPUT_CHANNEL_NUM[0], + # Model configs. + model_depth=cfg.RESNET.DEPTH, + model_num_class=cfg.MODEL.NUM_CLASSES, + dropout_rate=cfg.MODEL.DROPOUT_RATE, + # Normalization configs. + norm=get_norm(cfg), + norm_eps=1e-5, + norm_momentum=0.1, + # Activation configs. + activation=partial(nn.ReLU, inplace=cfg.RESNET.INPLACE_RELU), + # Stem configs. + stem_dim_out=cfg.RESNET.WIDTH_PER_GROUP, + stem_conv_kernel_size=(1, 7, 7), + stem_conv_stride=(1, 2, 2), + # Stage configs. + stage_conv_a_kernel_size=( + (1, 1, 1), + (1, 1, 1), + (1, 1, 1), + (1, 1, 1), + ), + stage_conv_b_kernel_size=( + (3, 3, 3), + (3, 3, 3), + (3, 3, 3), + (3, 3, 3), + ), + stage_conv_b_num_groups=(1, 1, 1, 1), + stage_conv_b_dilation=( + (1, 1, 1), + (1, 1, 1), + (1, 1, 1), + (1, 1, 1), + ), + stage_spatial_stride=(2, 2, 2, 2), + stage_temporal_stride=(1, 1, 2, 2), + stage_bottleneck=( + create_2plus1d_bottleneck_block, + create_2plus1d_bottleneck_block, + create_2plus1d_bottleneck_block, + create_2plus1d_bottleneck_block, + ), + # Head configs. + head_pool=nn.AvgPool3d, + head_pool_kernel_size=( + cfg.DATA.NUM_FRAMES // 4, + cfg.DATA.TRAIN_CROP_SIZE // 32, + cfg.DATA.TRAIN_CROP_SIZE // 32, + ), + head_activation=None, + head_output_with_global_average=False, + ) + + self.post_act = get_head_act(cfg.MODEL.HEAD_ACT) + + def forward(self, x, bboxes=None): + x = x[0] + x = self.model(x) + # Performs fully convlutional inference. + if not self.training: + x = self.post_act(x) + x = x.mean([2, 3, 4]) + + x = x.view(x.shape[0], -1) + return x diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/resnet_helper.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/resnet_helper.py" new file mode 100644 index 0000000..d31df70 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/resnet_helper.py" @@ -0,0 +1,725 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Video models.""" + +import torch +import torch.nn as nn + +from slowfast.models.common import drop_path +from slowfast.models.nonlocal_helper import Nonlocal +from slowfast.models.operators import SE, Swish + + +def get_trans_func(name): + """ + Retrieves the transformation module by name. + """ + trans_funcs = { + "bottleneck_transform": BottleneckTransform, + "basic_transform": BasicTransform, + "x3d_transform": X3DTransform, + } + assert ( + name in trans_funcs.keys() + ), "Transformation function '{}' not supported".format(name) + return trans_funcs[name] + + +class BasicTransform(nn.Module): + """ + Basic transformation: Tx3x3, 1x3x3, where T is the size of temporal kernel. + """ + + def __init__( + self, + dim_in, + dim_out, + temp_kernel_size, + stride, + dim_inner=None, + num_groups=1, + stride_1x1=None, + inplace_relu=True, + eps=1e-5, + bn_mmt=0.1, + norm_module=nn.BatchNorm3d, + block_idx=0, + ): + """ + Args: + dim_in (int): the channel dimensions of the input. + dim_out (int): the channel dimension of the output. + temp_kernel_size (int): the temporal kernel sizes of the first + convolution in the basic block. + stride (int): the stride of the bottleneck. + dim_inner (None): the inner dimension would not be used in + BasicTransform. + num_groups (int): number of groups for the convolution. Number of + group is always 1 for BasicTransform. + stride_1x1 (None): stride_1x1 will not be used in BasicTransform. + inplace_relu (bool): if True, calculate the relu on the original + input without allocating new memory. + eps (float): epsilon for batch norm. + bn_mmt (float): momentum for batch norm. Noted that BN momentum in + PyTorch = 1 - BN momentum in Caffe2. + norm_module (nn.Module): nn.Module for the normalization layer. The + default is nn.BatchNorm3d. + """ + super(BasicTransform, self).__init__() + self.temp_kernel_size = temp_kernel_size + self._inplace_relu = inplace_relu + self._eps = eps + self._bn_mmt = bn_mmt + self._construct(dim_in, dim_out, stride, norm_module) + + def _construct(self, dim_in, dim_out, stride, norm_module): + # Tx3x3, BN, ReLU. + self.a = nn.Conv3d( + dim_in, + dim_out, + kernel_size=[self.temp_kernel_size, 3, 3], + stride=[1, stride, stride], + padding=[int(self.temp_kernel_size // 2), 1, 1], + bias=False, + ) + self.a_bn = norm_module( + num_features=dim_out, eps=self._eps, momentum=self._bn_mmt + ) + self.a_relu = nn.ReLU(inplace=self._inplace_relu) + # 1x3x3, BN. + self.b = nn.Conv3d( + dim_out, + dim_out, + kernel_size=[1, 3, 3], + stride=[1, 1, 1], + padding=[0, 1, 1], + bias=False, + ) + self.b_bn = norm_module( + num_features=dim_out, eps=self._eps, momentum=self._bn_mmt + ) + + self.b_bn.transform_final_bn = True + + def forward(self, x): + x = self.a(x) + x = self.a_bn(x) + x = self.a_relu(x) + + x = self.b(x) + x = self.b_bn(x) + return x + + +class X3DTransform(nn.Module): + """ + X3D transformation: 1x1x1, Tx3x3 (channelwise, num_groups=dim_in), 1x1x1, + augmented with (optional) SE (squeeze-excitation) on the 3x3x3 output. + T is the temporal kernel size (defaulting to 3) + """ + + def __init__( + self, + dim_in, + dim_out, + temp_kernel_size, + stride, + dim_inner, + num_groups, + stride_1x1=False, + inplace_relu=True, + eps=1e-5, + bn_mmt=0.1, + dilation=1, + norm_module=nn.BatchNorm3d, + se_ratio=0.0625, + swish_inner=True, + block_idx=0, + ): + """ + Args: + dim_in (int): the channel dimensions of the input. + dim_out (int): the channel dimension of the output. + temp_kernel_size (int): the temporal kernel sizes of the middle + convolution in the bottleneck. + stride (int): the stride of the bottleneck. + dim_inner (int): the inner dimension of the block. + num_groups (int): number of groups for the convolution. num_groups=1 + is for standard ResNet like networks, and num_groups>1 is for + ResNeXt like networks. + stride_1x1 (bool): if True, apply stride to 1x1 conv, otherwise + apply stride to the 3x3 conv. + inplace_relu (bool): if True, calculate the relu on the original + input without allocating new memory. + eps (float): epsilon for batch norm. + bn_mmt (float): momentum for batch norm. Noted that BN momentum in + PyTorch = 1 - BN momentum in Caffe2. + dilation (int): size of dilation. + norm_module (nn.Module): nn.Module for the normalization layer. The + default is nn.BatchNorm3d. + se_ratio (float): if > 0, apply SE to the Tx3x3 conv, with the SE + channel dimensionality being se_ratio times the Tx3x3 conv dim. + swish_inner (bool): if True, apply swish to the Tx3x3 conv, otherwise + apply ReLU to the Tx3x3 conv. + """ + super(X3DTransform, self).__init__() + self.temp_kernel_size = temp_kernel_size + self._inplace_relu = inplace_relu + self._eps = eps + self._bn_mmt = bn_mmt + self._se_ratio = se_ratio + self._swish_inner = swish_inner + self._stride_1x1 = stride_1x1 + self._block_idx = block_idx + self._construct( + dim_in, + dim_out, + stride, + dim_inner, + num_groups, + dilation, + norm_module, + ) + + def _construct( + self, + dim_in, + dim_out, + stride, + dim_inner, + num_groups, + dilation, + norm_module, + ): + (str1x1, str3x3) = (stride, 1) if self._stride_1x1 else (1, stride) + + # 1x1x1, BN, ReLU. + self.a = nn.Conv3d( + dim_in, + dim_inner, + kernel_size=[1, 1, 1], + stride=[1, str1x1, str1x1], + padding=[0, 0, 0], + bias=False, + ) + self.a_bn = norm_module( + num_features=dim_inner, eps=self._eps, momentum=self._bn_mmt + ) + self.a_relu = nn.ReLU(inplace=self._inplace_relu) + + # Tx3x3, BN, ReLU. + self.b = nn.Conv3d( + dim_inner, + dim_inner, + [self.temp_kernel_size, 3, 3], + stride=[1, str3x3, str3x3], + padding=[int(self.temp_kernel_size // 2), dilation, dilation], + groups=num_groups, + bias=False, + dilation=[1, dilation, dilation], + ) + self.b_bn = norm_module( + num_features=dim_inner, eps=self._eps, momentum=self._bn_mmt + ) + + # Apply SE attention or not + use_se = True if (self._block_idx + 1) % 2 else False + if self._se_ratio > 0.0 and use_se: + self.se = SE(dim_inner, self._se_ratio) + + if self._swish_inner: + self.b_relu = Swish() + else: + self.b_relu = nn.ReLU(inplace=self._inplace_relu) + + # 1x1x1, BN. + self.c = nn.Conv3d( + dim_inner, + dim_out, + kernel_size=[1, 1, 1], + stride=[1, 1, 1], + padding=[0, 0, 0], + bias=False, + ) + self.c_bn = norm_module( + num_features=dim_out, eps=self._eps, momentum=self._bn_mmt + ) + self.c_bn.transform_final_bn = True + + def forward(self, x): + for block in self.children(): + x = block(x) + return x + + +class BottleneckTransform(nn.Module): + """ + Bottleneck transformation: Tx1x1, 1x3x3, 1x1x1, where T is the size of + temporal kernel. + """ + + def __init__( + self, + dim_in, + dim_out, + temp_kernel_size, + stride, + dim_inner, + num_groups, + stride_1x1=False, + inplace_relu=True, + eps=1e-5, + bn_mmt=0.1, + dilation=1, + norm_module=nn.BatchNorm3d, + block_idx=0, + ): + """ + Args: + dim_in (int): the channel dimensions of the input. + dim_out (int): the channel dimension of the output. + temp_kernel_size (int): the temporal kernel sizes of the first + convolution in the bottleneck. + stride (int): the stride of the bottleneck. + dim_inner (int): the inner dimension of the block. + num_groups (int): number of groups for the convolution. num_groups=1 + is for standard ResNet like networks, and num_groups>1 is for + ResNeXt like networks. + stride_1x1 (bool): if True, apply stride to 1x1 conv, otherwise + apply stride to the 3x3 conv. + inplace_relu (bool): if True, calculate the relu on the original + input without allocating new memory. + eps (float): epsilon for batch norm. + bn_mmt (float): momentum for batch norm. Noted that BN momentum in + PyTorch = 1 - BN momentum in Caffe2. + dilation (int): size of dilation. + norm_module (nn.Module): nn.Module for the normalization layer. The + default is nn.BatchNorm3d. + """ + super(BottleneckTransform, self).__init__() + self.temp_kernel_size = temp_kernel_size + self._inplace_relu = inplace_relu + self._eps = eps + self._bn_mmt = bn_mmt + self._stride_1x1 = stride_1x1 + self._construct( + dim_in, + dim_out, + stride, + dim_inner, + num_groups, + dilation, + norm_module, + ) + + def _construct( + self, + dim_in, + dim_out, + stride, + dim_inner, + num_groups, + dilation, + norm_module, + ): + (str1x1, str3x3) = (stride, 1) if self._stride_1x1 else (1, stride) + + # Tx1x1, BN, ReLU. + self.a = nn.Conv3d( + dim_in, + dim_inner, + kernel_size=[self.temp_kernel_size, 1, 1], + stride=[1, str1x1, str1x1], + padding=[int(self.temp_kernel_size // 2), 0, 0], + bias=False, + ) + self.a_bn = norm_module( + num_features=dim_inner, eps=self._eps, momentum=self._bn_mmt + ) + self.a_relu = nn.ReLU(inplace=self._inplace_relu) + + # 1x3x3, BN, ReLU. + self.b = nn.Conv3d( + dim_inner, + dim_inner, + [1, 3, 3], + stride=[1, str3x3, str3x3], + padding=[0, dilation, dilation], + groups=num_groups, + bias=False, + dilation=[1, dilation, dilation], + ) + self.b_bn = norm_module( + num_features=dim_inner, eps=self._eps, momentum=self._bn_mmt + ) + self.b_relu = nn.ReLU(inplace=self._inplace_relu) + + # 1x1x1, BN. + self.c = nn.Conv3d( + dim_inner, + dim_out, + kernel_size=[1, 1, 1], + stride=[1, 1, 1], + padding=[0, 0, 0], + bias=False, + ) + self.c_bn = norm_module( + num_features=dim_out, eps=self._eps, momentum=self._bn_mmt + ) + self.c_bn.transform_final_bn = True + + def forward(self, x): + # Explicitly forward every layer. + # Branch2a. + x = self.a(x) + x = self.a_bn(x) + x = self.a_relu(x) + + # Branch2b. + x = self.b(x) + x = self.b_bn(x) + x = self.b_relu(x) + + # Branch2c + x = self.c(x) + x = self.c_bn(x) + return x + + +class ResBlock(nn.Module): + """ + Residual block. + """ + + def __init__( + self, + dim_in, + dim_out, + temp_kernel_size, + stride, + trans_func, + dim_inner, + num_groups=1, + stride_1x1=False, + inplace_relu=True, + eps=1e-5, + bn_mmt=0.1, + dilation=1, + norm_module=nn.BatchNorm3d, + block_idx=0, + drop_connect_rate=0.0, + ): + """ + ResBlock class constructs redisual blocks. More details can be found in: + Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun. + "Deep residual learning for image recognition." + https://arxiv.org/abs/1512.03385 + Args: + dim_in (int): the channel dimensions of the input. + dim_out (int): the channel dimension of the output. + temp_kernel_size (int): the temporal kernel sizes of the middle + convolution in the bottleneck. + stride (int): the stride of the bottleneck. + trans_func (string): transform function to be used to construct the + bottleneck. + dim_inner (int): the inner dimension of the block. + num_groups (int): number of groups for the convolution. num_groups=1 + is for standard ResNet like networks, and num_groups>1 is for + ResNeXt like networks. + stride_1x1 (bool): if True, apply stride to 1x1 conv, otherwise + apply stride to the 3x3 conv. + inplace_relu (bool): calculate the relu on the original input + without allocating new memory. + eps (float): epsilon for batch norm. + bn_mmt (float): momentum for batch norm. Noted that BN momentum in + PyTorch = 1 - BN momentum in Caffe2. + dilation (int): size of dilation. + norm_module (nn.Module): nn.Module for the normalization layer. The + default is nn.BatchNorm3d. + drop_connect_rate (float): basic rate at which blocks are dropped, + linearly increases from input to output blocks. + """ + super(ResBlock, self).__init__() + self._inplace_relu = inplace_relu + self._eps = eps + self._bn_mmt = bn_mmt + self._drop_connect_rate = drop_connect_rate + self._construct( + dim_in, + dim_out, + temp_kernel_size, + stride, + trans_func, + dim_inner, + num_groups, + stride_1x1, + inplace_relu, + dilation, + norm_module, + block_idx, + ) + + def _construct( + self, + dim_in, + dim_out, + temp_kernel_size, + stride, + trans_func, + dim_inner, + num_groups, + stride_1x1, + inplace_relu, + dilation, + norm_module, + block_idx, + ): + # Use skip connection with projection if dim or res change. + if (dim_in != dim_out) or (stride != 1): + self.branch1 = nn.Conv3d( + dim_in, + dim_out, + kernel_size=1, + stride=[1, stride, stride], + padding=0, + bias=False, + dilation=1, + ) + self.branch1_bn = norm_module( + num_features=dim_out, eps=self._eps, momentum=self._bn_mmt + ) + self.branch2 = trans_func( + dim_in, + dim_out, + temp_kernel_size, + stride, + dim_inner, + num_groups, + stride_1x1=stride_1x1, + inplace_relu=inplace_relu, + dilation=dilation, + norm_module=norm_module, + block_idx=block_idx, + ) + self.relu = nn.ReLU(self._inplace_relu) + + def forward(self, x): + f_x = self.branch2(x) + if self.training and self._drop_connect_rate > 0.0: + f_x = drop_path(f_x, self._drop_connect_rate) + if hasattr(self, "branch1"): + x = self.branch1_bn(self.branch1(x)) + f_x + else: + x = x + f_x + x = self.relu(x) + return x + + +class ResStage(nn.Module): + """ + Stage of 3D ResNet. It expects to have one or more tensors as input for + single pathway (C2D, I3D, Slow), and multi-pathway (SlowFast) cases. + More details can be found here: + + Christoph Feichtenhofer, Haoqi Fan, Jitendra Malik, and Kaiming He. + "SlowFast networks for video recognition." + https://arxiv.org/pdf/1812.03982.pdf + """ + + def __init__( + self, + dim_in, + dim_out, + stride, + temp_kernel_sizes, + num_blocks, + dim_inner, + num_groups, + num_block_temp_kernel, + nonlocal_inds, + nonlocal_group, + nonlocal_pool, + dilation, + instantiation="softmax", + trans_func_name="bottleneck_transform", + stride_1x1=False, + inplace_relu=True, + norm_module=nn.BatchNorm3d, + drop_connect_rate=0.0, + ): + """ + The `__init__` method of any subclass should also contain these arguments. + ResStage builds p streams, where p can be greater or equal to one. + Args: + dim_in (list): list of p the channel dimensions of the input. + Different channel dimensions control the input dimension of + different pathways. + dim_out (list): list of p the channel dimensions of the output. + Different channel dimensions control the input dimension of + different pathways. + temp_kernel_sizes (list): list of the p temporal kernel sizes of the + convolution in the bottleneck. Different temp_kernel_sizes + control different pathway. + stride (list): list of the p strides of the bottleneck. Different + stride control different pathway. + num_blocks (list): list of p numbers of blocks for each of the + pathway. + dim_inner (list): list of the p inner channel dimensions of the + input. Different channel dimensions control the input dimension + of different pathways. + num_groups (list): list of number of p groups for the convolution. + num_groups=1 is for standard ResNet like networks, and + num_groups>1 is for ResNeXt like networks. + num_block_temp_kernel (list): extent the temp_kernel_sizes to + num_block_temp_kernel blocks, then fill temporal kernel size + of 1 for the rest of the layers. + nonlocal_inds (list): If the tuple is empty, no nonlocal layer will + be added. If the tuple is not empty, add nonlocal layers after + the index-th block. + dilation (list): size of dilation for each pathway. + nonlocal_group (list): list of number of p nonlocal groups. Each + number controls how to fold temporal dimension to batch + dimension before applying nonlocal transformation. + https://github.com/facebookresearch/video-nonlocal-net. + instantiation (string): different instantiation for nonlocal layer. + Supports two different instantiation method: + "dot_product": normalizing correlation matrix with L2. + "softmax": normalizing correlation matrix with Softmax. + trans_func_name (string): name of the the transformation function apply + on the network. + norm_module (nn.Module): nn.Module for the normalization layer. The + default is nn.BatchNorm3d. + drop_connect_rate (float): basic rate at which blocks are dropped, + linearly increases from input to output blocks. + """ + super(ResStage, self).__init__() + assert all( + ( + num_block_temp_kernel[i] <= num_blocks[i] + for i in range(len(temp_kernel_sizes)) + ) + ) + self.num_blocks = num_blocks + self.nonlocal_group = nonlocal_group + self._drop_connect_rate = drop_connect_rate + self.temp_kernel_sizes = [ + (temp_kernel_sizes[i] * num_blocks[i])[: num_block_temp_kernel[i]] + + [1] * (num_blocks[i] - num_block_temp_kernel[i]) + for i in range(len(temp_kernel_sizes)) + ] + assert ( + len( + { + len(dim_in), + len(dim_out), + len(temp_kernel_sizes), + len(stride), + len(num_blocks), + len(dim_inner), + len(num_groups), + len(num_block_temp_kernel), + len(nonlocal_inds), + len(nonlocal_group), + } + ) + == 1 + ) + self.num_pathways = len(self.num_blocks) + self._construct( + dim_in, + dim_out, + stride, + dim_inner, + num_groups, + trans_func_name, + stride_1x1, + inplace_relu, + nonlocal_inds, + nonlocal_pool, + instantiation, + dilation, + norm_module, + ) + + def _construct( + self, + dim_in, + dim_out, + stride, + dim_inner, + num_groups, + trans_func_name, + stride_1x1, + inplace_relu, + nonlocal_inds, + nonlocal_pool, + instantiation, + dilation, + norm_module, + ): + for pathway in range(self.num_pathways): + for i in range(self.num_blocks[pathway]): + # Retrieve the transformation function. + trans_func = get_trans_func(trans_func_name) + # Construct the block. + res_block = ResBlock( + dim_in[pathway] if i == 0 else dim_out[pathway], + dim_out[pathway], + self.temp_kernel_sizes[pathway][i], + stride[pathway] if i == 0 else 1, + trans_func, + dim_inner[pathway], + num_groups[pathway], + stride_1x1=stride_1x1, + inplace_relu=inplace_relu, + dilation=dilation[pathway], + norm_module=norm_module, + block_idx=i, + drop_connect_rate=self._drop_connect_rate, + ) + self.add_module("pathway{}_res{}".format(pathway, i), res_block) + if i in nonlocal_inds[pathway]: + nln = Nonlocal( + dim_out[pathway], + dim_out[pathway] // 2, + nonlocal_pool[pathway], + instantiation=instantiation, + norm_module=norm_module, + ) + self.add_module( + "pathway{}_nonlocal{}".format(pathway, i), nln + ) + + def forward(self, inputs): + output = [] + for pathway in range(self.num_pathways): + x = inputs[pathway] + for i in range(self.num_blocks[pathway]): + m = getattr(self, "pathway{}_res{}".format(pathway, i)) + x = m(x) + if hasattr(self, "pathway{}_nonlocal{}".format(pathway, i)): + nln = getattr( + self, "pathway{}_nonlocal{}".format(pathway, i) + ) + b, c, t, h, w = x.shape + if self.nonlocal_group[pathway] > 1: + # Fold temporal dimension into batch dimension. + x = x.permute(0, 2, 1, 3, 4) + x = x.reshape( + b * self.nonlocal_group[pathway], + t // self.nonlocal_group[pathway], + c, + h, + w, + ) + x = x.permute(0, 2, 1, 3, 4) + x = nln(x) + if self.nonlocal_group[pathway] > 1: + # Fold back to temporal dimension. + x = x.permute(0, 2, 1, 3, 4) + x = x.reshape(b, t, c, h, w) + x = x.permute(0, 2, 1, 3, 4) + output.append(x) + + return output diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/stem_helper.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/stem_helper.py" new file mode 100644 index 0000000..f4d2861 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/stem_helper.py" @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""ResNe(X)t 3D stem helper.""" + +import torch.nn as nn + + +def get_stem_func(name): + """ + Retrieves the stem module by name. + """ + trans_funcs = {"x3d_stem": X3DStem, "basic_stem": ResNetBasicStem} + assert ( + name in trans_funcs.keys() + ), "Transformation function '{}' not supported".format(name) + return trans_funcs[name] + + +class VideoModelStem(nn.Module): + """ + Video 3D stem module. Provides stem operations of Conv, BN, ReLU, MaxPool + on input data tensor for one or multiple pathways. + """ + + def __init__( + self, + dim_in, + dim_out, + kernel, + stride, + padding, + inplace_relu=True, + eps=1e-5, + bn_mmt=0.1, + norm_module=nn.BatchNorm3d, + stem_func_name="basic_stem", + ): + """ + The `__init__` method of any subclass should also contain these + arguments. List size of 1 for single pathway models (C2D, I3D, Slow + and etc), list size of 2 for two pathway models (SlowFast). + + Args: + dim_in (list): the list of channel dimensions of the inputs. + dim_out (list): the output dimension of the convolution in the stem + layer. + kernel (list): the kernels' size of the convolutions in the stem + layers. Temporal kernel size, height kernel size, width kernel + size in order. + stride (list): the stride sizes of the convolutions in the stem + layer. Temporal kernel stride, height kernel size, width kernel + size in order. + padding (list): the paddings' sizes of the convolutions in the stem + layer. Temporal padding size, height padding size, width padding + size in order. + inplace_relu (bool): calculate the relu on the original input + without allocating new memory. + eps (float): epsilon for batch norm. + bn_mmt (float): momentum for batch norm. Noted that BN momentum in + PyTorch = 1 - BN momentum in Caffe2. + norm_module (nn.Module): nn.Module for the normalization layer. The + default is nn.BatchNorm3d. + stem_func_name (string): name of the the stem function applied on + input to the network. + """ + super(VideoModelStem, self).__init__() + + assert ( + len( + { + len(dim_in), + len(dim_out), + len(kernel), + len(stride), + len(padding), + } + ) + == 1 + ), "Input pathway dimensions are not consistent. {} {} {} {} {}".format( + len(dim_in), + len(dim_out), + len(kernel), + len(stride), + len(padding), + ) + + self.num_pathways = len(dim_in) + self.kernel = kernel + self.stride = stride + self.padding = padding + self.inplace_relu = inplace_relu + self.eps = eps + self.bn_mmt = bn_mmt + # Construct the stem layer. + self._construct_stem(dim_in, dim_out, norm_module, stem_func_name) + + def _construct_stem(self, dim_in, dim_out, norm_module, stem_func_name): + trans_func = get_stem_func(stem_func_name) + + for pathway in range(len(dim_in)): + stem = trans_func( + dim_in[pathway], + dim_out[pathway], + self.kernel[pathway], + self.stride[pathway], + self.padding[pathway], + self.inplace_relu, + self.eps, + self.bn_mmt, + norm_module, + ) + self.add_module("pathway{}_stem".format(pathway), stem) + + def forward(self, x): + assert ( + len(x) == self.num_pathways + ), "Input tensor does not contain {} pathway".format(self.num_pathways) + for pathway in range(len(x)): + m = getattr(self, "pathway{}_stem".format(pathway)) + x[pathway] = m(x[pathway]) + return x + + +class ResNetBasicStem(nn.Module): + """ + ResNe(X)t 3D stem module. + Performs spatiotemporal Convolution, BN, and Relu following by a + spatiotemporal pooling. + """ + + def __init__( + self, + dim_in, + dim_out, + kernel, + stride, + padding, + inplace_relu=True, + eps=1e-5, + bn_mmt=0.1, + norm_module=nn.BatchNorm3d, + ): + """ + The `__init__` method of any subclass should also contain these arguments. + + Args: + dim_in (int): the channel dimension of the input. Normally 3 is used + for rgb input, and 2 or 3 is used for optical flow input. + dim_out (int): the output dimension of the convolution in the stem + layer. + kernel (list): the kernel size of the convolution in the stem layer. + temporal kernel size, height kernel size, width kernel size in + order. + stride (list): the stride size of the convolution in the stem layer. + temporal kernel stride, height kernel size, width kernel size in + order. + padding (int): the padding size of the convolution in the stem + layer, temporal padding size, height padding size, width + padding size in order. + inplace_relu (bool): calculate the relu on the original input + without allocating new memory. + eps (float): epsilon for batch norm. + bn_mmt (float): momentum for batch norm. Noted that BN momentum in + PyTorch = 1 - BN momentum in Caffe2. + norm_module (nn.Module): nn.Module for the normalization layer. The + default is nn.BatchNorm3d. + """ + super(ResNetBasicStem, self).__init__() + self.kernel = kernel + self.stride = stride + self.padding = padding + self.inplace_relu = inplace_relu + self.eps = eps + self.bn_mmt = bn_mmt + # Construct the stem layer. + self._construct_stem(dim_in, dim_out, norm_module) + + def _construct_stem(self, dim_in, dim_out, norm_module): + self.conv = nn.Conv3d( + dim_in, + dim_out, + self.kernel, + stride=self.stride, + padding=self.padding, + bias=False, + ) + self.bn = norm_module( + num_features=dim_out, eps=self.eps, momentum=self.bn_mmt + ) + self.relu = nn.ReLU(self.inplace_relu) + self.pool_layer = nn.MaxPool3d( + kernel_size=[1, 3, 3], stride=[1, 2, 2], padding=[0, 1, 1] + ) + + def forward(self, x): + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + x = self.pool_layer(x) + return x + + +class X3DStem(nn.Module): + """ + X3D's 3D stem module. + Performs a spatial followed by a depthwise temporal Convolution, BN, and Relu following by a + spatiotemporal pooling. + """ + + def __init__( + self, + dim_in, + dim_out, + kernel, + stride, + padding, + inplace_relu=True, + eps=1e-5, + bn_mmt=0.1, + norm_module=nn.BatchNorm3d, + ): + """ + The `__init__` method of any subclass should also contain these arguments. + + Args: + dim_in (int): the channel dimension of the input. Normally 3 is used + for rgb input, and 2 or 3 is used for optical flow input. + dim_out (int): the output dimension of the convolution in the stem + layer. + kernel (list): the kernel size of the convolution in the stem layer. + temporal kernel size, height kernel size, width kernel size in + order. + stride (list): the stride size of the convolution in the stem layer. + temporal kernel stride, height kernel size, width kernel size in + order. + padding (int): the padding size of the convolution in the stem + layer, temporal padding size, height padding size, width + padding size in order. + inplace_relu (bool): calculate the relu on the original input + without allocating new memory. + eps (float): epsilon for batch norm. + bn_mmt (float): momentum for batch norm. Noted that BN momentum in + PyTorch = 1 - BN momentum in Caffe2. + norm_module (nn.Module): nn.Module for the normalization layer. The + default is nn.BatchNorm3d. + """ + super(X3DStem, self).__init__() + self.kernel = kernel + self.stride = stride + self.padding = padding + self.inplace_relu = inplace_relu + self.eps = eps + self.bn_mmt = bn_mmt + # Construct the stem layer. + self._construct_stem(dim_in, dim_out, norm_module) + + def _construct_stem(self, dim_in, dim_out, norm_module): + self.conv_xy = nn.Conv3d( + dim_in, + dim_out, + kernel_size=(1, self.kernel[1], self.kernel[2]), + stride=(1, self.stride[1], self.stride[2]), + padding=(0, self.padding[1], self.padding[2]), + bias=False, + ) + self.conv = nn.Conv3d( + dim_out, + dim_out, + kernel_size=(self.kernel[0], 1, 1), + stride=(self.stride[0], 1, 1), + padding=(self.padding[0], 0, 0), + bias=False, + groups=dim_out, + ) + + self.bn = norm_module( + num_features=dim_out, eps=self.eps, momentum=self.bn_mmt + ) + self.relu = nn.ReLU(self.inplace_relu) + + def forward(self, x): + x = self.conv_xy(x) + x = self.conv(x) + x = self.bn(x) + x = self.relu(x) + return x + + +class PatchEmbed(nn.Module): + """ + PatchEmbed. + """ + + def __init__( + self, + dim_in=3, + dim_out=768, + kernel=(1, 16, 16), + stride=(1, 4, 4), + padding=(1, 7, 7), + conv_2d=False, + ): + super().__init__() + if conv_2d: + conv = nn.Conv2d + else: + conv = nn.Conv3d + self.proj = conv( + dim_in, + dim_out, + kernel_size=kernel, + stride=stride, + padding=padding, + ) + + def forward(self, x): + x = self.proj(x) + # B C (T) H W -> B (T)HW C + return x.flatten(2).transpose(1, 2) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/utils.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/utils.py" new file mode 100644 index 0000000..988110f --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/utils.py" @@ -0,0 +1,21 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import slowfast.utils.logging as logging + +logger = logging.get_logger(__name__) + + +def round_width(width, multiplier, min_width=1, divisor=1, verbose=False): + if not multiplier: + return width + width *= multiplier + min_width = min_width or divisor + if verbose: + logger.info(f"min width {min_width}") + logger.info(f"width {width} divisor {divisor}") + logger.info(f"other {int(width + divisor / 2) // divisor * divisor}") + + width_out = max(min_width, int(width + divisor / 2) // divisor * divisor) + if width_out < 0.9 * width: + width_out += divisor + return int(width_out) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/video_model_builder.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/video_model_builder.py" new file mode 100644 index 0000000..4c3de6e --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/models/video_model_builder.py" @@ -0,0 +1,1016 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + + +"""Video models.""" + +import math +from functools import partial +import torch +import torch.nn as nn +from torch.nn.init import trunc_normal_ + +import slowfast.utils.weight_init_helper as init_helper +from slowfast.models.attention import MultiScaleBlock +from slowfast.models.batchnorm_helper import get_norm +from slowfast.models.stem_helper import PatchEmbed +from slowfast.models.utils import round_width + +from . import head_helper, resnet_helper, stem_helper +from .build import MODEL_REGISTRY + +# Number of blocks for different stages given the model depth. +_MODEL_STAGE_DEPTH = {50: (3, 4, 6, 3), 101: (3, 4, 23, 3)} + +# Basis of temporal kernel sizes for each of the stage. +_TEMPORAL_KERNEL_BASIS = { + "2d": [ + [[1]], # conv1 temporal kernel. + [[1]], # res2 temporal kernel. + [[1]], # res3 temporal kernel. + [[1]], # res4 temporal kernel. + [[1]], # res5 temporal kernel. + ], + "c2d": [ + [[1]], # conv1 temporal kernel. + [[1]], # res2 temporal kernel. + [[1]], # res3 temporal kernel. + [[1]], # res4 temporal kernel. + [[1]], # res5 temporal kernel. + ], + "c2d_nopool": [ + [[1]], # conv1 temporal kernel. + [[1]], # res2 temporal kernel. + [[1]], # res3 temporal kernel. + [[1]], # res4 temporal kernel. + [[1]], # res5 temporal kernel. + ], + "i3d": [ + [[5]], # conv1 temporal kernel. + [[3]], # res2 temporal kernel. + [[3, 1]], # res3 temporal kernel. + [[3, 1]], # res4 temporal kernel. + [[1, 3]], # res5 temporal kernel. + ], + "i3d_nopool": [ + [[5]], # conv1 temporal kernel. + [[3]], # res2 temporal kernel. + [[3, 1]], # res3 temporal kernel. + [[3, 1]], # res4 temporal kernel. + [[1, 3]], # res5 temporal kernel. + ], + "slow": [ + [[1]], # conv1 temporal kernel. + [[1]], # res2 temporal kernel. + [[1]], # res3 temporal kernel. + [[3]], # res4 temporal kernel. + [[3]], # res5 temporal kernel. + ], + "slowfast": [ + [[1], [5]], # conv1 temporal kernel for slow and fast pathway. + [[1], [3]], # res2 temporal kernel for slow and fast pathway. + [[1], [3]], # res3 temporal kernel for slow and fast pathway. + [[3], [3]], # res4 temporal kernel for slow and fast pathway. + [[3], [3]], # res5 temporal kernel for slow and fast pathway. + ], + "x3d": [ + [[5]], # conv1 temporal kernels. + [[3]], # res2 temporal kernels. + [[3]], # res3 temporal kernels. + [[3]], # res4 temporal kernels. + [[3]], # res5 temporal kernels. + ], +} + +_POOL1 = { + "2d": [[1, 1, 1]], + "c2d": [[2, 1, 1]], + "c2d_nopool": [[1, 1, 1]], + "i3d": [[2, 1, 1]], + "i3d_nopool": [[1, 1, 1]], + "slow": [[1, 1, 1]], + "slowfast": [[1, 1, 1], [1, 1, 1]], + "x3d": [[1, 1, 1]], +} + + +class FuseFastToSlow(nn.Module): + """ + Fuses the information from the Fast pathway to the Slow pathway. Given the + tensors from Slow pathway and Fast pathway, fuse information from Fast to + Slow, then return the fused tensors from Slow and Fast pathway in order. + """ + + def __init__( + self, + dim_in, + fusion_conv_channel_ratio, + fusion_kernel, + alpha, + eps=1e-5, + bn_mmt=0.1, + inplace_relu=True, + norm_module=nn.BatchNorm3d, + ): + """ + Args: + dim_in (int): the channel dimension of the input. + fusion_conv_channel_ratio (int): channel ratio for the convolution + used to fuse from Fast pathway to Slow pathway. + fusion_kernel (int): kernel size of the convolution used to fuse + from Fast pathway to Slow pathway. + alpha (int): the frame rate ratio between the Fast and Slow pathway. + eps (float): epsilon for batch norm. + bn_mmt (float): momentum for batch norm. Noted that BN momentum in + PyTorch = 1 - BN momentum in Caffe2. + inplace_relu (bool): if True, calculate the relu on the original + input without allocating new memory. + norm_module (nn.Module): nn.Module for the normalization layer. The + default is nn.BatchNorm3d. + """ + super(FuseFastToSlow, self).__init__() + self.conv_f2s = nn.Conv3d( + dim_in, + dim_in * fusion_conv_channel_ratio, + kernel_size=[fusion_kernel, 1, 1], + stride=[alpha, 1, 1], + padding=[fusion_kernel // 2, 0, 0], + bias=False, + ) + self.bn = norm_module( + num_features=dim_in * fusion_conv_channel_ratio, + eps=eps, + momentum=bn_mmt, + ) + self.relu = nn.ReLU(inplace_relu) + + def forward(self, x): + x_s = x[0] + x_f = x[1] + fuse = self.conv_f2s(x_f) + fuse = self.bn(fuse) + fuse = self.relu(fuse) + x_s_fuse = torch.cat([x_s, fuse], 1) + return [x_s_fuse, x_f] + + +@MODEL_REGISTRY.register() +class SlowFast(nn.Module): + """ + SlowFast model builder for SlowFast network. + + Christoph Feichtenhofer, Haoqi Fan, Jitendra Malik, and Kaiming He. + "SlowFast networks for video recognition." + https://arxiv.org/pdf/1812.03982.pdf + """ + + def __init__(self, cfg): + """ + The `__init__` method of any subclass should also contain these + arguments. + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + super(SlowFast, self).__init__() + self.norm_module = get_norm(cfg) + self.enable_detection = cfg.DETECTION.ENABLE + self.num_pathways = 2 + self._construct_network(cfg) + init_helper.init_weights( + self, cfg.MODEL.FC_INIT_STD, cfg.RESNET.ZERO_INIT_FINAL_BN + ) + + def _construct_network(self, cfg): + """ + Builds a SlowFast model. The first pathway is the Slow pathway and the + second pathway is the Fast pathway. + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + assert cfg.MODEL.ARCH in _POOL1.keys() + pool_size = _POOL1[cfg.MODEL.ARCH] + assert len({len(pool_size), self.num_pathways}) == 1 + assert cfg.RESNET.DEPTH in _MODEL_STAGE_DEPTH.keys() + + (d2, d3, d4, d5) = _MODEL_STAGE_DEPTH[cfg.RESNET.DEPTH] + + num_groups = cfg.RESNET.NUM_GROUPS + width_per_group = cfg.RESNET.WIDTH_PER_GROUP + dim_inner = num_groups * width_per_group + out_dim_ratio = ( + cfg.SLOWFAST.BETA_INV // cfg.SLOWFAST.FUSION_CONV_CHANNEL_RATIO + ) + + temp_kernel = _TEMPORAL_KERNEL_BASIS[cfg.MODEL.ARCH] + + self.s1 = stem_helper.VideoModelStem( + dim_in=cfg.DATA.INPUT_CHANNEL_NUM, + dim_out=[width_per_group, width_per_group // cfg.SLOWFAST.BETA_INV], + kernel=[temp_kernel[0][0] + [7, 7], temp_kernel[0][1] + [7, 7]], + stride=[[1, 2, 2]] * 2, + padding=[ + [temp_kernel[0][0][0] // 2, 3, 3], + [temp_kernel[0][1][0] // 2, 3, 3], + ], + norm_module=self.norm_module, + ) + self.s1_fuse = FuseFastToSlow( + width_per_group // cfg.SLOWFAST.BETA_INV, + cfg.SLOWFAST.FUSION_CONV_CHANNEL_RATIO, + cfg.SLOWFAST.FUSION_KERNEL_SZ, + cfg.SLOWFAST.ALPHA, + norm_module=self.norm_module, + ) + + self.s2 = resnet_helper.ResStage( + dim_in=[ + width_per_group + width_per_group // out_dim_ratio, + width_per_group // cfg.SLOWFAST.BETA_INV, + ], + dim_out=[ + width_per_group * 4, + width_per_group * 4 // cfg.SLOWFAST.BETA_INV, + ], + dim_inner=[dim_inner, dim_inner // cfg.SLOWFAST.BETA_INV], + temp_kernel_sizes=temp_kernel[1], + stride=cfg.RESNET.SPATIAL_STRIDES[0], + num_blocks=[d2] * 2, + num_groups=[num_groups] * 2, + num_block_temp_kernel=cfg.RESNET.NUM_BLOCK_TEMP_KERNEL[0], + nonlocal_inds=cfg.NONLOCAL.LOCATION[0], + nonlocal_group=cfg.NONLOCAL.GROUP[0], + nonlocal_pool=cfg.NONLOCAL.POOL[0], + instantiation=cfg.NONLOCAL.INSTANTIATION, + trans_func_name=cfg.RESNET.TRANS_FUNC, + dilation=cfg.RESNET.SPATIAL_DILATIONS[0], + norm_module=self.norm_module, + ) + self.s2_fuse = FuseFastToSlow( + width_per_group * 4 // cfg.SLOWFAST.BETA_INV, + cfg.SLOWFAST.FUSION_CONV_CHANNEL_RATIO, + cfg.SLOWFAST.FUSION_KERNEL_SZ, + cfg.SLOWFAST.ALPHA, + norm_module=self.norm_module, + ) + + for pathway in range(self.num_pathways): + pool = nn.MaxPool3d( + kernel_size=pool_size[pathway], + stride=pool_size[pathway], + padding=[0, 0, 0], + ) + self.add_module("pathway{}_pool".format(pathway), pool) + + self.s3 = resnet_helper.ResStage( + dim_in=[ + width_per_group * 4 + width_per_group * 4 // out_dim_ratio, + width_per_group * 4 // cfg.SLOWFAST.BETA_INV, + ], + dim_out=[ + width_per_group * 8, + width_per_group * 8 // cfg.SLOWFAST.BETA_INV, + ], + dim_inner=[dim_inner * 2, dim_inner * 2 // cfg.SLOWFAST.BETA_INV], + temp_kernel_sizes=temp_kernel[2], + stride=cfg.RESNET.SPATIAL_STRIDES[1], + num_blocks=[d3] * 2, + num_groups=[num_groups] * 2, + num_block_temp_kernel=cfg.RESNET.NUM_BLOCK_TEMP_KERNEL[1], + nonlocal_inds=cfg.NONLOCAL.LOCATION[1], + nonlocal_group=cfg.NONLOCAL.GROUP[1], + nonlocal_pool=cfg.NONLOCAL.POOL[1], + instantiation=cfg.NONLOCAL.INSTANTIATION, + trans_func_name=cfg.RESNET.TRANS_FUNC, + dilation=cfg.RESNET.SPATIAL_DILATIONS[1], + norm_module=self.norm_module, + ) + self.s3_fuse = FuseFastToSlow( + width_per_group * 8 // cfg.SLOWFAST.BETA_INV, + cfg.SLOWFAST.FUSION_CONV_CHANNEL_RATIO, + cfg.SLOWFAST.FUSION_KERNEL_SZ, + cfg.SLOWFAST.ALPHA, + norm_module=self.norm_module, + ) + + self.s4 = resnet_helper.ResStage( + dim_in=[ + width_per_group * 8 + width_per_group * 8 // out_dim_ratio, + width_per_group * 8 // cfg.SLOWFAST.BETA_INV, + ], + dim_out=[ + width_per_group * 16, + width_per_group * 16 // cfg.SLOWFAST.BETA_INV, + ], + dim_inner=[dim_inner * 4, dim_inner * 4 // cfg.SLOWFAST.BETA_INV], + temp_kernel_sizes=temp_kernel[3], + stride=cfg.RESNET.SPATIAL_STRIDES[2], + num_blocks=[d4] * 2, + num_groups=[num_groups] * 2, + num_block_temp_kernel=cfg.RESNET.NUM_BLOCK_TEMP_KERNEL[2], + nonlocal_inds=cfg.NONLOCAL.LOCATION[2], + nonlocal_group=cfg.NONLOCAL.GROUP[2], + nonlocal_pool=cfg.NONLOCAL.POOL[2], + instantiation=cfg.NONLOCAL.INSTANTIATION, + trans_func_name=cfg.RESNET.TRANS_FUNC, + dilation=cfg.RESNET.SPATIAL_DILATIONS[2], + norm_module=self.norm_module, + ) + self.s4_fuse = FuseFastToSlow( + width_per_group * 16 // cfg.SLOWFAST.BETA_INV, + cfg.SLOWFAST.FUSION_CONV_CHANNEL_RATIO, + cfg.SLOWFAST.FUSION_KERNEL_SZ, + cfg.SLOWFAST.ALPHA, + norm_module=self.norm_module, + ) + + self.s5 = resnet_helper.ResStage( + dim_in=[ + width_per_group * 16 + width_per_group * 16 // out_dim_ratio, + width_per_group * 16 // cfg.SLOWFAST.BETA_INV, + ], + dim_out=[ + width_per_group * 32, + width_per_group * 32 // cfg.SLOWFAST.BETA_INV, + ], + dim_inner=[dim_inner * 8, dim_inner * 8 // cfg.SLOWFAST.BETA_INV], + temp_kernel_sizes=temp_kernel[4], + stride=cfg.RESNET.SPATIAL_STRIDES[3], + num_blocks=[d5] * 2, + num_groups=[num_groups] * 2, + num_block_temp_kernel=cfg.RESNET.NUM_BLOCK_TEMP_KERNEL[3], + nonlocal_inds=cfg.NONLOCAL.LOCATION[3], + nonlocal_group=cfg.NONLOCAL.GROUP[3], + nonlocal_pool=cfg.NONLOCAL.POOL[3], + instantiation=cfg.NONLOCAL.INSTANTIATION, + trans_func_name=cfg.RESNET.TRANS_FUNC, + dilation=cfg.RESNET.SPATIAL_DILATIONS[3], + norm_module=self.norm_module, + ) + + if cfg.DETECTION.ENABLE: + self.head = head_helper.ResNetRoIHead( + dim_in=[ + width_per_group * 32, + width_per_group * 32 // cfg.SLOWFAST.BETA_INV, + ], + num_classes=cfg.MODEL.NUM_CLASSES, + pool_size=[ + [ + cfg.DATA.NUM_FRAMES + // cfg.SLOWFAST.ALPHA + // pool_size[0][0], + 1, + 1, + ], + [cfg.DATA.NUM_FRAMES // pool_size[1][0], 1, 1], + ], + resolution=[[cfg.DETECTION.ROI_XFORM_RESOLUTION] * 2] * 2, + scale_factor=[cfg.DETECTION.SPATIAL_SCALE_FACTOR] * 2, + dropout_rate=cfg.MODEL.DROPOUT_RATE, + act_func=cfg.MODEL.HEAD_ACT, + aligned=cfg.DETECTION.ALIGNED, + ) + else: + self.head = head_helper.ResNetBasicHead( + dim_in=[ + width_per_group * 32, + width_per_group * 32 // cfg.SLOWFAST.BETA_INV, + ], + num_classes=cfg.MODEL.NUM_CLASSES, + pool_size=[None, None] + if cfg.MULTIGRID.SHORT_CYCLE + else [ + [ + cfg.DATA.NUM_FRAMES + // cfg.SLOWFAST.ALPHA + // pool_size[0][0], + cfg.DATA.TRAIN_CROP_SIZE // 32 // pool_size[0][1], + cfg.DATA.TRAIN_CROP_SIZE // 32 // pool_size[0][2], + ], + [ + cfg.DATA.NUM_FRAMES // pool_size[1][0], + cfg.DATA.TRAIN_CROP_SIZE // 32 // pool_size[1][1], + cfg.DATA.TRAIN_CROP_SIZE // 32 // pool_size[1][2], + ], + ], # None for AdaptiveAvgPool3d((1, 1, 1)) + dropout_rate=cfg.MODEL.DROPOUT_RATE, + act_func=cfg.MODEL.HEAD_ACT, + ) + + def forward(self, x, bboxes=None): + x = self.s1(x) + x = self.s1_fuse(x) + x = self.s2(x) + x = self.s2_fuse(x) + for pathway in range(self.num_pathways): + pool = getattr(self, "pathway{}_pool".format(pathway)) + x[pathway] = pool(x[pathway]) + x = self.s3(x) + x = self.s3_fuse(x) + x = self.s4(x) + x = self.s4_fuse(x) + x = self.s5(x) + if self.enable_detection: + x = self.head(x, bboxes) + else: + x = self.head(x) + return x + + +@MODEL_REGISTRY.register() +class ResNet(nn.Module): + """ + ResNet model builder. It builds a ResNet like network backbone without + lateral connection (C2D, I3D, Slow). + + Christoph Feichtenhofer, Haoqi Fan, Jitendra Malik, and Kaiming He. + "SlowFast networks for video recognition." + https://arxiv.org/pdf/1812.03982.pdf + + Xiaolong Wang, Ross Girshick, Abhinav Gupta, and Kaiming He. + "Non-local neural networks." + https://arxiv.org/pdf/1711.07971.pdf + """ + + def __init__(self, cfg): + """ + The `__init__` method of any subclass should also contain these + arguments. + + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + super(ResNet, self).__init__() + self.norm_module = get_norm(cfg) + self.enable_detection = cfg.DETECTION.ENABLE + self.num_pathways = 1 + self._construct_network(cfg) + init_helper.init_weights( + self, cfg.MODEL.FC_INIT_STD, cfg.RESNET.ZERO_INIT_FINAL_BN + ) + + def _construct_network(self, cfg): + """ + Builds a single pathway ResNet model. + + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + assert cfg.MODEL.ARCH in _POOL1.keys() + pool_size = _POOL1[cfg.MODEL.ARCH] + assert len({len(pool_size), self.num_pathways}) == 1 + assert cfg.RESNET.DEPTH in _MODEL_STAGE_DEPTH.keys() + + (d2, d3, d4, d5) = _MODEL_STAGE_DEPTH[cfg.RESNET.DEPTH] + + num_groups = cfg.RESNET.NUM_GROUPS + width_per_group = cfg.RESNET.WIDTH_PER_GROUP + dim_inner = num_groups * width_per_group + + temp_kernel = _TEMPORAL_KERNEL_BASIS[cfg.MODEL.ARCH] + + self.s1 = stem_helper.VideoModelStem( + dim_in=cfg.DATA.INPUT_CHANNEL_NUM, + dim_out=[width_per_group], + kernel=[temp_kernel[0][0] + [7, 7]], + stride=[[1, 2, 2]], + padding=[[temp_kernel[0][0][0] // 2, 3, 3]], + norm_module=self.norm_module, + ) + + self.s2 = resnet_helper.ResStage( + dim_in=[width_per_group], + dim_out=[width_per_group * 4], + dim_inner=[dim_inner], + temp_kernel_sizes=temp_kernel[1], + stride=cfg.RESNET.SPATIAL_STRIDES[0], + num_blocks=[d2], + num_groups=[num_groups], + num_block_temp_kernel=cfg.RESNET.NUM_BLOCK_TEMP_KERNEL[0], + nonlocal_inds=cfg.NONLOCAL.LOCATION[0], + nonlocal_group=cfg.NONLOCAL.GROUP[0], + nonlocal_pool=cfg.NONLOCAL.POOL[0], + instantiation=cfg.NONLOCAL.INSTANTIATION, + trans_func_name=cfg.RESNET.TRANS_FUNC, + stride_1x1=cfg.RESNET.STRIDE_1X1, + inplace_relu=cfg.RESNET.INPLACE_RELU, + dilation=cfg.RESNET.SPATIAL_DILATIONS[0], + norm_module=self.norm_module, + ) + + for pathway in range(self.num_pathways): + pool = nn.MaxPool3d( + kernel_size=pool_size[pathway], + stride=pool_size[pathway], + padding=[0, 0, 0], + ) + self.add_module("pathway{}_pool".format(pathway), pool) + + self.s3 = resnet_helper.ResStage( + dim_in=[width_per_group * 4], + dim_out=[width_per_group * 8], + dim_inner=[dim_inner * 2], + temp_kernel_sizes=temp_kernel[2], + stride=cfg.RESNET.SPATIAL_STRIDES[1], + num_blocks=[d3], + num_groups=[num_groups], + num_block_temp_kernel=cfg.RESNET.NUM_BLOCK_TEMP_KERNEL[1], + nonlocal_inds=cfg.NONLOCAL.LOCATION[1], + nonlocal_group=cfg.NONLOCAL.GROUP[1], + nonlocal_pool=cfg.NONLOCAL.POOL[1], + instantiation=cfg.NONLOCAL.INSTANTIATION, + trans_func_name=cfg.RESNET.TRANS_FUNC, + stride_1x1=cfg.RESNET.STRIDE_1X1, + inplace_relu=cfg.RESNET.INPLACE_RELU, + dilation=cfg.RESNET.SPATIAL_DILATIONS[1], + norm_module=self.norm_module, + ) + + self.s4 = resnet_helper.ResStage( + dim_in=[width_per_group * 8], + dim_out=[width_per_group * 16], + dim_inner=[dim_inner * 4], + temp_kernel_sizes=temp_kernel[3], + stride=cfg.RESNET.SPATIAL_STRIDES[2], + num_blocks=[d4], + num_groups=[num_groups], + num_block_temp_kernel=cfg.RESNET.NUM_BLOCK_TEMP_KERNEL[2], + nonlocal_inds=cfg.NONLOCAL.LOCATION[2], + nonlocal_group=cfg.NONLOCAL.GROUP[2], + nonlocal_pool=cfg.NONLOCAL.POOL[2], + instantiation=cfg.NONLOCAL.INSTANTIATION, + trans_func_name=cfg.RESNET.TRANS_FUNC, + stride_1x1=cfg.RESNET.STRIDE_1X1, + inplace_relu=cfg.RESNET.INPLACE_RELU, + dilation=cfg.RESNET.SPATIAL_DILATIONS[2], + norm_module=self.norm_module, + ) + + self.s5 = resnet_helper.ResStage( + dim_in=[width_per_group * 16], + dim_out=[width_per_group * 32], + dim_inner=[dim_inner * 8], + temp_kernel_sizes=temp_kernel[4], + stride=cfg.RESNET.SPATIAL_STRIDES[3], + num_blocks=[d5], + num_groups=[num_groups], + num_block_temp_kernel=cfg.RESNET.NUM_BLOCK_TEMP_KERNEL[3], + nonlocal_inds=cfg.NONLOCAL.LOCATION[3], + nonlocal_group=cfg.NONLOCAL.GROUP[3], + nonlocal_pool=cfg.NONLOCAL.POOL[3], + instantiation=cfg.NONLOCAL.INSTANTIATION, + trans_func_name=cfg.RESNET.TRANS_FUNC, + stride_1x1=cfg.RESNET.STRIDE_1X1, + inplace_relu=cfg.RESNET.INPLACE_RELU, + dilation=cfg.RESNET.SPATIAL_DILATIONS[3], + norm_module=self.norm_module, + ) + + if self.enable_detection: + self.head = head_helper.ResNetRoIHead( + dim_in=[width_per_group * 32], + num_classes=cfg.MODEL.NUM_CLASSES, + pool_size=[[cfg.DATA.NUM_FRAMES // pool_size[0][0], 1, 1]], + resolution=[[cfg.DETECTION.ROI_XFORM_RESOLUTION] * 2], + scale_factor=[cfg.DETECTION.SPATIAL_SCALE_FACTOR], + dropout_rate=cfg.MODEL.DROPOUT_RATE, + act_func=cfg.MODEL.HEAD_ACT, + aligned=cfg.DETECTION.ALIGNED, + ) + else: + self.head = head_helper.ResNetBasicHead( + dim_in=[width_per_group * 32], + num_classes=cfg.MODEL.NUM_CLASSES, + pool_size=[None, None] + if cfg.MULTIGRID.SHORT_CYCLE + else [ + [ + cfg.DATA.NUM_FRAMES // pool_size[0][0], + cfg.DATA.TRAIN_CROP_SIZE // 32 // pool_size[0][1], + cfg.DATA.TRAIN_CROP_SIZE // 32 // pool_size[0][2], + ] + ], # None for AdaptiveAvgPool3d((1, 1, 1)) + dropout_rate=cfg.MODEL.DROPOUT_RATE, + act_func=cfg.MODEL.HEAD_ACT, + ) + + def forward(self, x, bboxes=None): + x = self.s1(x) + x = self.s2(x) + for pathway in range(self.num_pathways): + pool = getattr(self, "pathway{}_pool".format(pathway)) + x[pathway] = pool(x[pathway]) + x = self.s3(x) + x = self.s4(x) + x = self.s5(x) + if self.enable_detection: + x = self.head(x, bboxes) + else: + x = self.head(x) + return x + + +@MODEL_REGISTRY.register() +class X3D(nn.Module): + """ + X3D model builder. It builds a X3D network backbone, which is a ResNet. + + Christoph Feichtenhofer. + "X3D: Expanding Architectures for Efficient Video Recognition." + https://arxiv.org/abs/2004.04730 + """ + + def __init__(self, cfg): + """ + The `__init__` method of any subclass should also contain these + arguments. + + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + super(X3D, self).__init__() + self.norm_module = get_norm(cfg) + self.enable_detection = cfg.DETECTION.ENABLE + self.num_pathways = 1 + + exp_stage = 2.0 + self.dim_c1 = cfg.X3D.DIM_C1 + + self.dim_res2 = ( + round_width(self.dim_c1, exp_stage, divisor=8) + if cfg.X3D.SCALE_RES2 + else self.dim_c1 + ) + self.dim_res3 = round_width(self.dim_res2, exp_stage, divisor=8) + self.dim_res4 = round_width(self.dim_res3, exp_stage, divisor=8) + self.dim_res5 = round_width(self.dim_res4, exp_stage, divisor=8) + + self.block_basis = [ + # blocks, c, stride + [1, self.dim_res2, 2], + [2, self.dim_res3, 2], + [5, self.dim_res4, 2], + [3, self.dim_res5, 2], + ] + self._construct_network(cfg) + init_helper.init_weights( + self, cfg.MODEL.FC_INIT_STD, cfg.RESNET.ZERO_INIT_FINAL_BN + ) + + def _round_repeats(self, repeats, multiplier): + """Round number of layers based on depth multiplier.""" + multiplier = multiplier + if not multiplier: + return repeats + return int(math.ceil(multiplier * repeats)) + + def _construct_network(self, cfg): + """ + Builds a single pathway X3D model. + + Args: + cfg (CfgNode): model building configs, details are in the + comments of the config file. + """ + assert cfg.MODEL.ARCH in _POOL1.keys() + assert cfg.RESNET.DEPTH in _MODEL_STAGE_DEPTH.keys() + + (d2, d3, d4, d5) = _MODEL_STAGE_DEPTH[cfg.RESNET.DEPTH] + + num_groups = cfg.RESNET.NUM_GROUPS + width_per_group = cfg.RESNET.WIDTH_PER_GROUP + dim_inner = num_groups * width_per_group + + w_mul = cfg.X3D.WIDTH_FACTOR + d_mul = cfg.X3D.DEPTH_FACTOR + dim_res1 = round_width(self.dim_c1, w_mul) + + temp_kernel = _TEMPORAL_KERNEL_BASIS[cfg.MODEL.ARCH] + + self.s1 = stem_helper.VideoModelStem( + dim_in=cfg.DATA.INPUT_CHANNEL_NUM, + dim_out=[dim_res1], + kernel=[temp_kernel[0][0] + [3, 3]], + stride=[[1, 2, 2]], + padding=[[temp_kernel[0][0][0] // 2, 1, 1]], + norm_module=self.norm_module, + stem_func_name="x3d_stem", + ) + + # blob_in = s1 + dim_in = dim_res1 + for stage, block in enumerate(self.block_basis): + dim_out = round_width(block[1], w_mul) + dim_inner = int(cfg.X3D.BOTTLENECK_FACTOR * dim_out) + + n_rep = self._round_repeats(block[0], d_mul) + prefix = "s{}".format( + stage + 2 + ) # start w res2 to follow convention + + s = resnet_helper.ResStage( + dim_in=[dim_in], + dim_out=[dim_out], + dim_inner=[dim_inner], + temp_kernel_sizes=temp_kernel[1], + stride=[block[2]], + num_blocks=[n_rep], + num_groups=[dim_inner] + if cfg.X3D.CHANNELWISE_3x3x3 + else [num_groups], + num_block_temp_kernel=[n_rep], + nonlocal_inds=cfg.NONLOCAL.LOCATION[0], + nonlocal_group=cfg.NONLOCAL.GROUP[0], + nonlocal_pool=cfg.NONLOCAL.POOL[0], + instantiation=cfg.NONLOCAL.INSTANTIATION, + trans_func_name=cfg.RESNET.TRANS_FUNC, + stride_1x1=cfg.RESNET.STRIDE_1X1, + norm_module=self.norm_module, + dilation=cfg.RESNET.SPATIAL_DILATIONS[stage], + drop_connect_rate=cfg.MODEL.DROPCONNECT_RATE + * (stage + 2) + / (len(self.block_basis) + 1), + ) + dim_in = dim_out + self.add_module(prefix, s) + + if self.enable_detection: + NotImplementedError + else: + spat_sz = int(math.ceil(cfg.DATA.TRAIN_CROP_SIZE / 32.0)) + self.head = head_helper.X3DHead( + dim_in=dim_out, + dim_inner=dim_inner, + dim_out=cfg.X3D.DIM_C5, + num_classes=cfg.MODEL.NUM_CLASSES, + pool_size=[cfg.DATA.NUM_FRAMES, spat_sz, spat_sz], + dropout_rate=cfg.MODEL.DROPOUT_RATE, + act_func=cfg.MODEL.HEAD_ACT, + bn_lin5_on=cfg.X3D.BN_LIN5, + ) + + def forward(self, x, bboxes=None): + for module in self.children(): + x = module(x) + return x + + +@MODEL_REGISTRY.register() +class MViT(nn.Module): + """ + Multiscale Vision Transformers + Haoqi Fan, Bo Xiong, Karttikeya Mangalam, Yanghao Li, Zhicheng Yan, Jitendra Malik, Christoph Feichtenhofer + https://arxiv.org/abs/2104.11227 + """ + + def __init__(self, cfg): + super().__init__() + # Get parameters. + assert cfg.DATA.TRAIN_CROP_SIZE == cfg.DATA.TEST_CROP_SIZE + self.cfg = cfg + pool_first = cfg.MVIT.POOL_FIRST + # Prepare input. + spatial_size = cfg.DATA.TRAIN_CROP_SIZE + temporal_size = cfg.DATA.NUM_FRAMES + in_chans = cfg.DATA.INPUT_CHANNEL_NUM[0] + use_2d_patch = cfg.MVIT.PATCH_2D + self.patch_stride = cfg.MVIT.PATCH_STRIDE + if use_2d_patch: + self.patch_stride = [1] + self.patch_stride + # Prepare output. + num_classes = cfg.MODEL.NUM_CLASSES + embed_dim = cfg.MVIT.EMBED_DIM + # Prepare backbone + num_heads = cfg.MVIT.NUM_HEADS + mlp_ratio = cfg.MVIT.MLP_RATIO + qkv_bias = cfg.MVIT.QKV_BIAS + self.drop_rate = cfg.MVIT.DROPOUT_RATE + depth = cfg.MVIT.DEPTH + drop_path_rate = cfg.MVIT.DROPPATH_RATE + mode = cfg.MVIT.MODE + self.cls_embed_on = cfg.MVIT.CLS_EMBED_ON + self.sep_pos_embed = cfg.MVIT.SEP_POS_EMBED + if cfg.MVIT.NORM == "layernorm": + norm_layer = partial(nn.LayerNorm, eps=1e-6) + else: + raise NotImplementedError("Only supports layernorm.") + self.num_classes = num_classes + self.patch_embed = stem_helper.PatchEmbed( + dim_in=in_chans, + dim_out=embed_dim, + kernel=cfg.MVIT.PATCH_KERNEL, + stride=cfg.MVIT.PATCH_STRIDE, + padding=cfg.MVIT.PATCH_PADDING, + conv_2d=use_2d_patch, + ) + self.input_dims = [temporal_size, spatial_size, spatial_size] + assert self.input_dims[1] == self.input_dims[2] + self.patch_dims = [ + self.input_dims[i] // self.patch_stride[i] + for i in range(len(self.input_dims)) + ] + num_patches = math.prod(self.patch_dims) + + dpr = [ + x.item() for x in torch.linspace(0, drop_path_rate, depth) + ] # stochastic depth decay rule + + if self.cls_embed_on: + self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim)) + pos_embed_dim = num_patches + 1 + else: + pos_embed_dim = num_patches + + if self.sep_pos_embed: + self.pos_embed_spatial = nn.Parameter( + torch.zeros( + 1, self.patch_dims[1] * self.patch_dims[2], embed_dim + ) + ) + self.pos_embed_temporal = nn.Parameter( + torch.zeros(1, self.patch_dims[0], embed_dim) + ) + if self.cls_embed_on: + self.pos_embed_class = nn.Parameter(torch.zeros(1, 1, embed_dim)) + else: + self.pos_embed = nn.Parameter( + torch.zeros(1, pos_embed_dim, embed_dim) + ) + + if self.drop_rate > 0.0: + self.pos_drop = nn.Dropout(p=self.drop_rate) + + pool_q = [[] for i in range(cfg.MVIT.DEPTH)] + pool_kv = [[] for i in range(cfg.MVIT.DEPTH)] + stride_q = [[] for i in range(cfg.MVIT.DEPTH)] + stride_kv = [[] for i in range(cfg.MVIT.DEPTH)] + + for i in range(len(cfg.MVIT.POOL_Q_STRIDE)): + stride_q[cfg.MVIT.POOL_Q_STRIDE[i][0]] = cfg.MVIT.POOL_Q_STRIDE[i][ + 1: + ] + if cfg.MVIT.POOL_KVQ_KERNEL is not None: + pool_q[ + cfg.MVIT.POOL_Q_STRIDE[i][0] + ] = cfg.MVIT.POOL_KVQ_KERNEL + else: + pool_q[cfg.MVIT.POOL_Q_STRIDE[i][0]] = [ + s + 1 if s > 1 else s for s in cfg.MVIT.POOL_Q_STRIDE[i][1:] + ] + for i in range(len(cfg.MVIT.POOL_KV_STRIDE)): + stride_kv[cfg.MVIT.POOL_KV_STRIDE[i][0]] = cfg.MVIT.POOL_KV_STRIDE[ + i + ][1:] + if cfg.MVIT.POOL_KVQ_KERNEL is not None: + pool_kv[ + cfg.MVIT.POOL_KV_STRIDE[i][0] + ] = cfg.MVIT.POOL_KVQ_KERNEL + else: + pool_kv[cfg.MVIT.POOL_KV_STRIDE[i][0]] = [ + s + 1 if s > 1 else s + for s in cfg.MVIT.POOL_KV_STRIDE[i][1:] + ] + + dim_mul, head_mul = torch.ones(depth + 1), torch.ones(depth + 1) + for i in range(len(cfg.MVIT.DIM_MUL)): + dim_mul[cfg.MVIT.DIM_MUL[i][0]] = cfg.MVIT.DIM_MUL[i][1] + for i in range(len(cfg.MVIT.HEAD_MUL)): + head_mul[cfg.MVIT.HEAD_MUL[i][0]] = cfg.MVIT.HEAD_MUL[i][1] + + self.norm_stem = norm_layer(embed_dim) if cfg.MVIT.NORM_STEM else None + + self.blocks = nn.ModuleList() + for i in range(depth): + num_heads = round_width(num_heads, head_mul[i]) + embed_dim = round_width(embed_dim, dim_mul[i], divisor=num_heads) + dim_out = round_width( + embed_dim, + dim_mul[i + 1], + divisor=round_width(num_heads, head_mul[i + 1]), + ) + + self.blocks.append( + MultiScaleBlock( + dim=embed_dim, + dim_out=dim_out, + num_heads=num_heads, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + drop_rate=self.drop_rate, + drop_path=dpr[i], + norm_layer=norm_layer, + kernel_q=pool_q[i] if len(pool_q) > i else [], + kernel_kv=pool_kv[i] if len(pool_kv) > i else [], + stride_q=stride_q[i] if len(stride_q) > i else [], + stride_kv=stride_kv[i] if len(stride_kv) > i else [], + mode=mode, + has_cls_embed=self.cls_embed_on, + pool_first=pool_first, + ) + ) + + embed_dim = dim_out + self.norm = norm_layer(embed_dim) + + self.head = head_helper.TransformerBasicHead( + embed_dim, + num_classes, + dropout_rate=cfg.MODEL.DROPOUT_RATE, + act_func=cfg.MODEL.HEAD_ACT, + ) + if self.sep_pos_embed: + trunc_normal_(self.pos_embed_spatial, std=0.02) + trunc_normal_(self.pos_embed_temporal, std=0.02) + if self.cls_embed_on: + trunc_normal_(self.pos_embed_class, std=0.02) + else: + trunc_normal_(self.pos_embed, std=0.02) + if self.cls_embed_on: + trunc_normal_(self.cls_token, std=0.02) + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + nn.init.trunc_normal_(m.weight, std=0.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + + @torch.jit.ignore + def no_weight_decay(self): + if self.cfg.MVIT.ZERO_DECAY_POS_CLS: + if self.sep_pos_embed: + if self.cls_embed_on: + return { + "pos_embed_spatial", + "pos_embed_temporal", + "pos_embed_class", + "cls_token", + } + else: + return { + "pos_embed_spatial", + "pos_embed_temporal", + "pos_embed_class", + } + else: + if self.cls_embed_on: + return {"pos_embed", "cls_token"} + else: + return {"pos_embed"} + else: + return {} + + def forward(self, x): + x = x[0] + x = self.patch_embed(x) + + T = self.cfg.DATA.NUM_FRAMES // self.patch_stride[0] + H = self.cfg.DATA.TRAIN_CROP_SIZE // self.patch_stride[1] + W = self.cfg.DATA.TRAIN_CROP_SIZE // self.patch_stride[2] + B, N, C = x.shape + + if self.cls_embed_on: + cls_tokens = self.cls_token.expand( + B, -1, -1 + ) # stole cls_tokens impl from Phil Wang, thanks + x = torch.cat((cls_tokens, x), dim=1) + + if self.sep_pos_embed: + pos_embed = self.pos_embed_spatial.repeat( + 1, self.patch_dims[0], 1 + ) + torch.repeat_interleave( + self.pos_embed_temporal, + self.patch_dims[1] * self.patch_dims[2], + dim=1, + ) + if self.cls_embed_on: + pos_embed = torch.cat([self.pos_embed_class, pos_embed], 1) + x = x + pos_embed + else: + x = x + self.pos_embed + + if self.drop_rate: + x = self.pos_drop(x) + + if self.norm_stem: + x = self.norm_stem(x) + + thw = [T, H, W] + for blk in self.blocks: + x, thw = blk(x, thw) + + x = self.norm(x) + if self.cls_embed_on: + x = x[:, 0] + else: + x = x.mean(1) + + x = self.head(x) + return x diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/__init__.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/__init__.py" new file mode 100644 index 0000000..8dbe96a --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/__init__.py" @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_eval_helper.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_eval_helper.py" new file mode 100644 index 0000000..4bc3595 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_eval_helper.py" @@ -0,0 +1,304 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################## +# +# Based on: +# -------------------------------------------------------- +# ActivityNet +# Copyright (c) 2015 ActivityNet +# Licensed under The MIT License +# [see https://github.com/activitynet/ActivityNet/blob/master/LICENSE for details] +# -------------------------------------------------------- + +"""Helper functions for AVA evaluation.""" + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) +import csv +import logging +import numpy as np +import pprint +import time +from collections import defaultdict +from iopath.common.file_io import g_pathmgr + +import slowfast.utils.distributed as du +from slowfast.utils.ava_evaluation import ( + object_detection_evaluation, + standard_fields, +) + +logger = logging.getLogger(__name__) + + +def make_image_key(video_id, timestamp): + """Returns a unique identifier for a video id & timestamp.""" + return "%s,%04d" % (video_id, int(timestamp)) + + +def read_csv(csv_file, class_whitelist=None, load_score=False): + """Loads boxes and class labels from a CSV file in the AVA format. + CSV file format described at https://research.google.com/ava/download.html. + Args: + csv_file: A file object. + class_whitelist: If provided, boxes corresponding to (integer) class labels + not in this set are skipped. + Returns: + boxes: A dictionary mapping each unique image key (string) to a list of + boxes, given as coordinates [y1, x1, y2, x2]. + labels: A dictionary mapping each unique image key (string) to a list of + integer class lables, matching the corresponding box in `boxes`. + scores: A dictionary mapping each unique image key (string) to a list of + score values lables, matching the corresponding label in `labels`. If + scores are not provided in the csv, then they will default to 1.0. + """ + boxes = defaultdict(list) + labels = defaultdict(list) + scores = defaultdict(list) + with g_pathmgr.open(csv_file, "r") as f: + reader = csv.reader(f) + for row in reader: + assert len(row) in [7, 8], "Wrong number of columns: " + row + image_key = make_image_key(row[0], row[1]) + x1, y1, x2, y2 = [float(n) for n in row[2:6]] + action_id = int(row[6]) + if class_whitelist and action_id not in class_whitelist: + continue + score = 1.0 + if load_score: + score = float(row[7]) + boxes[image_key].append([y1, x1, y2, x2]) + labels[image_key].append(action_id) + scores[image_key].append(score) + return boxes, labels, scores + + +def read_exclusions(exclusions_file): + """Reads a CSV file of excluded timestamps. + Args: + exclusions_file: A file object containing a csv of video-id,timestamp. + Returns: + A set of strings containing excluded image keys, e.g. "aaaaaaaaaaa,0904", + or an empty set if exclusions file is None. + """ + excluded = set() + if exclusions_file: + with g_pathmgr.open(exclusions_file, "r") as f: + reader = csv.reader(f) + for row in reader: + assert len(row) == 2, "Expected only 2 columns, got: " + row + excluded.add(make_image_key(row[0], row[1])) + return excluded + + +def read_labelmap(labelmap_file): + """Read label map and class ids.""" + + labelmap = [] + class_ids = set() + name = "" + class_id = "" + with g_pathmgr.open(labelmap_file, "r") as f: + for line in f: + if line.startswith(" name:"): + name = line.split('"')[1] + elif line.startswith(" id:") or line.startswith(" label_id:"): + class_id = int(line.strip().split(" ")[-1]) + labelmap.append({"id": class_id, "name": name}) + class_ids.add(class_id) + return labelmap, class_ids + + +def evaluate_ava_from_files(labelmap, groundtruth, detections, exclusions): + """Run AVA evaluation given annotation/prediction files.""" + + categories, class_whitelist = read_labelmap(labelmap) + excluded_keys = read_exclusions(exclusions) + groundtruth = read_csv(groundtruth, class_whitelist, load_score=False) + detections = read_csv(detections, class_whitelist, load_score=True) + run_evaluation(categories, groundtruth, detections, excluded_keys) + + +def evaluate_ava( + preds, + original_boxes, + metadata, + excluded_keys, + class_whitelist, + categories, + groundtruth=None, + video_idx_to_name=None, + name="latest", +): + """Run AVA evaluation given numpy arrays.""" + + eval_start = time.time() + + detections = get_ava_eval_data( + preds, + original_boxes, + metadata, + class_whitelist, + video_idx_to_name=video_idx_to_name, + ) + + logger.info("Evaluating with %d unique GT frames." % len(groundtruth[0])) + logger.info( + "Evaluating with %d unique detection frames" % len(detections[0]) + ) + + write_results(detections, "detections_%s.csv" % name) + write_results(groundtruth, "groundtruth_%s.csv" % name) + + results = run_evaluation(categories, groundtruth, detections, excluded_keys) + + logger.info("AVA eval done in %f seconds." % (time.time() - eval_start)) + return results["PascalBoxes_Precision/mAP@0.5IOU"] + + +def run_evaluation( + categories, groundtruth, detections, excluded_keys, verbose=True +): + """AVA evaluation main logic.""" + + pascal_evaluator = object_detection_evaluation.PascalDetectionEvaluator( + categories + ) + + boxes, labels, _ = groundtruth + + gt_keys = [] + pred_keys = [] + + for image_key in boxes: + if image_key in excluded_keys: + logging.info( + ( + "Found excluded timestamp in ground truth: %s. " + "It will be ignored." + ), + image_key, + ) + continue + pascal_evaluator.add_single_ground_truth_image_info( + image_key, + { + standard_fields.InputDataFields.groundtruth_boxes: np.array( + boxes[image_key], dtype=float + ), + standard_fields.InputDataFields.groundtruth_classes: np.array( + labels[image_key], dtype=int + ), + standard_fields.InputDataFields.groundtruth_difficult: np.zeros( + len(boxes[image_key]), dtype=bool + ), + }, + ) + + gt_keys.append(image_key) + + boxes, labels, scores = detections + + for image_key in boxes: + if image_key in excluded_keys: + logging.info( + ( + "Found excluded timestamp in detections: %s. " + "It will be ignored." + ), + image_key, + ) + continue + pascal_evaluator.add_single_detected_image_info( + image_key, + { + standard_fields.DetectionResultFields.detection_boxes: np.array( + boxes[image_key], dtype=float + ), + standard_fields.DetectionResultFields.detection_classes: np.array( + labels[image_key], dtype=int + ), + standard_fields.DetectionResultFields.detection_scores: np.array( + scores[image_key], dtype=float + ), + }, + ) + + pred_keys.append(image_key) + + metrics = pascal_evaluator.evaluate() + + if du.is_master_proc(): + pprint.pprint(metrics, indent=2) + return metrics + + +def get_ava_eval_data( + scores, + boxes, + metadata, + class_whitelist, + verbose=False, + video_idx_to_name=None, +): + """ + Convert our data format into the data format used in official AVA + evaluation. + """ + + out_scores = defaultdict(list) + out_labels = defaultdict(list) + out_boxes = defaultdict(list) + count = 0 + for i in range(scores.shape[0]): + video_idx = int(np.round(metadata[i][0])) + sec = int(np.round(metadata[i][1])) + + video = video_idx_to_name[video_idx] + + key = video + "," + "%04d" % (sec) + batch_box = boxes[i].tolist() + # The first is batch idx. + batch_box = [batch_box[j] for j in [0, 2, 1, 4, 3]] + + one_scores = scores[i].tolist() + for cls_idx, score in enumerate(one_scores): + if cls_idx + 1 in class_whitelist: + out_scores[key].append(score) + out_labels[key].append(cls_idx + 1) + out_boxes[key].append(batch_box[1:]) + count += 1 + + return out_boxes, out_labels, out_scores + + +def write_results(detections, filename): + """Write prediction results into official formats.""" + start = time.time() + + boxes, labels, scores = detections + with g_pathmgr.open(filename, "w") as f: + for key in boxes.keys(): + for box, label, score in zip(boxes[key], labels[key], scores[key]): + f.write( + "%s,%.03f,%.03f,%.03f,%.03f,%d,%.04f\n" + % (key, box[1], box[0], box[3], box[2], label, score) + ) + + logger.info("AVA results wrote to %s" % filename) + logger.info("\ttook %d seconds." % (time.time() - start)) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/README.md" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/README.md" new file mode 100644 index 0000000..3fe4626 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/README.md" @@ -0,0 +1 @@ +The code under this folder is from the official [ActivityNet repo](https://github.com/activitynet/ActivityNet). diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/__init__.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/__init__.py" new file mode 100644 index 0000000..e69de29 diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/ava_action_list_v2.1_for_activitynet_2018.pbtxt.txt" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/ava_action_list_v2.1_for_activitynet_2018.pbtxt.txt" new file mode 100644 index 0000000..5e2c485 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/ava_action_list_v2.1_for_activitynet_2018.pbtxt.txt" @@ -0,0 +1,240 @@ +item { + name: "bend/bow (at the waist)" + id: 1 +} +item { + name: "crouch/kneel" + id: 3 +} +item { + name: "dance" + id: 4 +} +item { + name: "fall down" + id: 5 +} +item { + name: "get up" + id: 6 +} +item { + name: "jump/leap" + id: 7 +} +item { + name: "lie/sleep" + id: 8 +} +item { + name: "martial art" + id: 9 +} +item { + name: "run/jog" + id: 10 +} +item { + name: "sit" + id: 11 +} +item { + name: "stand" + id: 12 +} +item { + name: "swim" + id: 13 +} +item { + name: "walk" + id: 14 +} +item { + name: "answer phone" + id: 15 +} +item { + name: "carry/hold (an object)" + id: 17 +} +item { + name: "climb (e.g., a mountain)" + id: 20 +} +item { + name: "close (e.g., a door, a box)" + id: 22 +} +item { + name: "cut" + id: 24 +} +item { + name: "dress/put on clothing" + id: 26 +} +item { + name: "drink" + id: 27 +} +item { + name: "drive (e.g., a car, a truck)" + id: 28 +} +item { + name: "eat" + id: 29 +} +item { + name: "enter" + id: 30 +} +item { + name: "hit (an object)" + id: 34 +} +item { + name: "lift/pick up" + id: 36 +} +item { + name: "listen (e.g., to music)" + id: 37 +} +item { + name: "open (e.g., a window, a car door)" + id: 38 +} +item { + name: "play musical instrument" + id: 41 +} +item { + name: "point to (an object)" + id: 43 +} +item { + name: "pull (an object)" + id: 45 +} +item { + name: "push (an object)" + id: 46 +} +item { + name: "put down" + id: 47 +} +item { + name: "read" + id: 48 +} +item { + name: "ride (e.g., a bike, a car, a horse)" + id: 49 +} +item { + name: "sail boat" + id: 51 +} +item { + name: "shoot" + id: 52 +} +item { + name: "smoke" + id: 54 +} +item { + name: "take a photo" + id: 56 +} +item { + name: "text on/look at a cellphone" + id: 57 +} +item { + name: "throw" + id: 58 +} +item { + name: "touch (an object)" + id: 59 +} +item { + name: "turn (e.g., a screwdriver)" + id: 60 +} +item { + name: "watch (e.g., TV)" + id: 61 +} +item { + name: "work on a computer" + id: 62 +} +item { + name: "write" + id: 63 +} +item { + name: "fight/hit (a person)" + id: 64 +} +item { + name: "give/serve (an object) to (a person)" + id: 65 +} +item { + name: "grab (a person)" + id: 66 +} +item { + name: "hand clap" + id: 67 +} +item { + name: "hand shake" + id: 68 +} +item { + name: "hand wave" + id: 69 +} +item { + name: "hug (a person)" + id: 70 +} +item { + name: "kiss (a person)" + id: 72 +} +item { + name: "lift (a person)" + id: 73 +} +item { + name: "listen to (a person)" + id: 74 +} +item { + name: "push (another person)" + id: 76 +} +item { + name: "sing to (e.g., self, a person, a group)" + id: 77 +} +item { + name: "take (an object) from (a person)" + id: 78 +} +item { + name: "talk to (e.g., self, a person, a group)" + id: 79 +} +item { + name: "watch (a person)" + id: 80 +} diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/label_map_util.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/label_map_util.py" new file mode 100644 index 0000000..ce8933d --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/label_map_util.py" @@ -0,0 +1,187 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Label map utility functions.""" + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) +import logging + +# from google.protobuf import text_format +# from google3.third_party.tensorflow_models.object_detection.protos import string_int_label_map_pb2 + + +def _validate_label_map(label_map): + """Checks if a label map is valid. + + Args: + label_map: StringIntLabelMap to validate. + + Raises: + ValueError: if label map is invalid. + """ + for item in label_map.item: + if item.id < 1: + raise ValueError("Label map ids should be >= 1.") + + +def create_category_index(categories): + """Creates dictionary of COCO compatible categories keyed by category id. + + Args: + categories: a list of dicts, each of which has the following keys: + 'id': (required) an integer id uniquely identifying this category. + 'name': (required) string representing category name + e.g., 'cat', 'dog', 'pizza'. + + Returns: + category_index: a dict containing the same entries as categories, but keyed + by the 'id' field of each category. + """ + category_index = {} + for cat in categories: + category_index[cat["id"]] = cat + return category_index + + +def get_max_label_map_index(label_map): + """Get maximum index in label map. + + Args: + label_map: a StringIntLabelMapProto + + Returns: + an integer + """ + return max([item.id for item in label_map.item]) + + +def convert_label_map_to_categories( + label_map, max_num_classes, use_display_name=True +): + """Loads label map proto and returns categories list compatible with eval. + + This function loads a label map and returns a list of dicts, each of which + has the following keys: + 'id': (required) an integer id uniquely identifying this category. + 'name': (required) string representing category name + e.g., 'cat', 'dog', 'pizza'. + We only allow class into the list if its id-label_id_offset is + between 0 (inclusive) and max_num_classes (exclusive). + If there are several items mapping to the same id in the label map, + we will only keep the first one in the categories list. + + Args: + label_map: a StringIntLabelMapProto or None. If None, a default categories + list is created with max_num_classes categories. + max_num_classes: maximum number of (consecutive) label indices to include. + use_display_name: (boolean) choose whether to load 'display_name' field + as category name. If False or if the display_name field does not exist, + uses 'name' field as category names instead. + Returns: + categories: a list of dictionaries representing all possible categories. + """ + categories = [] + list_of_ids_already_added = [] + if not label_map: + label_id_offset = 1 + for class_id in range(max_num_classes): + categories.append( + { + "id": class_id + label_id_offset, + "name": "category_{}".format(class_id + label_id_offset), + } + ) + return categories + for item in label_map.item: + if not 0 < item.id <= max_num_classes: + logging.info( + "Ignore item %d since it falls outside of requested " + "label range.", + item.id, + ) + continue + if use_display_name and item.HasField("display_name"): + name = item.display_name + else: + name = item.name + if item.id not in list_of_ids_already_added: + list_of_ids_already_added.append(item.id) + categories.append({"id": item.id, "name": name}) + return categories + + +def load_labelmap(path): + """Loads label map proto. + + Args: + path: path to StringIntLabelMap proto text file. + Returns: + a StringIntLabelMapProto + """ + with open(path, "r") as fid: + label_map_string = fid.read() + label_map = string_int_label_map_pb2.StringIntLabelMap() + try: + text_format.Merge(label_map_string, label_map) + except text_format.ParseError: + label_map.ParseFromString(label_map_string) + _validate_label_map(label_map) + return label_map + + +def get_label_map_dict(label_map_path, use_display_name=False): + """Reads a label map and returns a dictionary of label names to id. + + Args: + label_map_path: path to label_map. + use_display_name: whether to use the label map items' display names as keys. + + Returns: + A dictionary mapping label names to id. + """ + label_map = load_labelmap(label_map_path) + label_map_dict = {} + for item in label_map.item: + if use_display_name: + label_map_dict[item.display_name] = item.id + else: + label_map_dict[item.name] = item.id + return label_map_dict + + +def create_category_index_from_labelmap(label_map_path): + """Reads a label map and returns a category index. + + Args: + label_map_path: Path to `StringIntLabelMap` proto text file. + + Returns: + A category index, which is a dictionary that maps integer ids to dicts + containing categories, e.g. + {1: {'id': 1, 'name': 'dog'}, 2: {'id': 2, 'name': 'cat'}, ...} + """ + label_map = load_labelmap(label_map_path) + max_num_classes = max(item.id for item in label_map.item) + categories = convert_label_map_to_categories(label_map, max_num_classes) + return create_category_index(categories) + + +def create_class_agnostic_category_index(): + """Creates a category index with a single `object` class.""" + return {1: {"id": 1, "name": "object"}} diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/metrics.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/metrics.py" new file mode 100644 index 0000000..c59a68d --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/metrics.py" @@ -0,0 +1,154 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Functions for computing metrics like precision, recall, CorLoc and etc.""" +from __future__ import division +import numpy as np + + +def compute_precision_recall(scores, labels, num_gt): + """Compute precision and recall. + + Args: + scores: A float numpy array representing detection score + labels: A boolean numpy array representing true/false positive labels + num_gt: Number of ground truth instances + + Raises: + ValueError: if the input is not of the correct format + + Returns: + precision: Fraction of positive instances over detected ones. This value is + None if no ground truth labels are present. + recall: Fraction of detected positive instance over all positive instances. + This value is None if no ground truth labels are present. + + """ + if ( + not isinstance(labels, np.ndarray) + or labels.dtype != np.bool + or len(labels.shape) != 1 + ): + raise ValueError("labels must be single dimension bool numpy array") + + if not isinstance(scores, np.ndarray) or len(scores.shape) != 1: + raise ValueError("scores must be single dimension numpy array") + + if num_gt < np.sum(labels): + raise ValueError( + "Number of true positives must be smaller than num_gt." + ) + + if len(scores) != len(labels): + raise ValueError("scores and labels must be of the same size.") + + if num_gt == 0: + return None, None + + sorted_indices = np.argsort(scores) + sorted_indices = sorted_indices[::-1] + labels = labels.astype(int) + true_positive_labels = labels[sorted_indices] + false_positive_labels = 1 - true_positive_labels + cum_true_positives = np.cumsum(true_positive_labels) + cum_false_positives = np.cumsum(false_positive_labels) + precision = cum_true_positives.astype(float) / ( + cum_true_positives + cum_false_positives + ) + recall = cum_true_positives.astype(float) / num_gt + return precision, recall + + +def compute_average_precision(precision, recall): + """Compute Average Precision according to the definition in VOCdevkit. + + Precision is modified to ensure that it does not decrease as recall + decrease. + + Args: + precision: A float [N, 1] numpy array of precisions + recall: A float [N, 1] numpy array of recalls + + Raises: + ValueError: if the input is not of the correct format + + Returns: + average_precison: The area under the precision recall curve. NaN if + precision and recall are None. + + """ + if precision is None: + if recall is not None: + raise ValueError("If precision is None, recall must also be None") + return np.NAN + + if not isinstance(precision, np.ndarray) or not isinstance( + recall, np.ndarray + ): + raise ValueError("precision and recall must be numpy array") + if precision.dtype != np.float or recall.dtype != np.float: + raise ValueError("input must be float numpy array.") + if len(precision) != len(recall): + raise ValueError("precision and recall must be of the same size.") + if not precision.size: + return 0.0 + if np.amin(precision) < 0 or np.amax(precision) > 1: + raise ValueError("Precision must be in the range of [0, 1].") + if np.amin(recall) < 0 or np.amax(recall) > 1: + raise ValueError("recall must be in the range of [0, 1].") + if not all(recall[i] <= recall[i + 1] for i in range(len(recall) - 1)): + raise ValueError("recall must be a non-decreasing array") + + recall = np.concatenate([[0], recall, [1]]) + precision = np.concatenate([[0], precision, [0]]) + + # Preprocess precision to be a non-decreasing array + for i in range(len(precision) - 2, -1, -1): + precision[i] = np.maximum(precision[i], precision[i + 1]) + + indices = np.where(recall[1:] != recall[:-1])[0] + 1 + average_precision = np.sum( + (recall[indices] - recall[indices - 1]) * precision[indices] + ) + return average_precision + + +def compute_cor_loc( + num_gt_imgs_per_class, num_images_correctly_detected_per_class +): + """Compute CorLoc according to the definition in the following paper. + + https://www.robots.ox.ac.uk/~vgg/rg/papers/deselaers-eccv10.pdf + + Returns nans if there are no ground truth images for a class. + + Args: + num_gt_imgs_per_class: 1D array, representing number of images containing + at least one object instance of a particular class + num_images_correctly_detected_per_class: 1D array, representing number of + images that are correctly detected at least one object instance of a + particular class + + Returns: + corloc_per_class: A float numpy array represents the corloc score of each + class + """ + # Divide by zero expected for classes with no gt examples. + with np.errstate(divide="ignore", invalid="ignore"): + return np.where( + num_gt_imgs_per_class == 0, + np.nan, + num_images_correctly_detected_per_class / num_gt_imgs_per_class, + ) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_list.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_list.py" new file mode 100644 index 0000000..a3bd38e --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_list.py" @@ -0,0 +1,143 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Numpy BoxList classes and functions.""" + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) +import numpy as np + + +class BoxList(object): + """Box collection. + + BoxList represents a list of bounding boxes as numpy array, where each + bounding box is represented as a row of 4 numbers, + [y_min, x_min, y_max, x_max]. It is assumed that all bounding boxes within a + given list correspond to a single image. + + Optionally, users can add additional related fields (such as + objectness/classification scores). + """ + + def __init__(self, data): + """Constructs box collection. + + Args: + data: a numpy array of shape [N, 4] representing box coordinates + + Raises: + ValueError: if bbox data is not a numpy array + ValueError: if invalid dimensions for bbox data + """ + if not isinstance(data, np.ndarray): + raise ValueError("data must be a numpy array.") + if len(data.shape) != 2 or data.shape[1] != 4: + raise ValueError("Invalid dimensions for box data.") + if data.dtype != np.float32 and data.dtype != np.float64: + raise ValueError( + "Invalid data type for box data: float is required." + ) + if not self._is_valid_boxes(data): + raise ValueError( + "Invalid box data. data must be a numpy array of " + "N*[y_min, x_min, y_max, x_max]" + ) + self.data = {"boxes": data} + + def num_boxes(self): + """Return number of boxes held in collections.""" + return self.data["boxes"].shape[0] + + def get_extra_fields(self): + """Return all non-box fields.""" + return [k for k in self.data.keys() if k != "boxes"] + + def has_field(self, field): + return field in self.data + + def add_field(self, field, field_data): + """Add data to a specified field. + + Args: + field: a string parameter used to speficy a related field to be accessed. + field_data: a numpy array of [N, ...] representing the data associated + with the field. + Raises: + ValueError: if the field is already exist or the dimension of the field + data does not matches the number of boxes. + """ + if self.has_field(field): + raise ValueError("Field " + field + "already exists") + if len(field_data.shape) < 1 or field_data.shape[0] != self.num_boxes(): + raise ValueError("Invalid dimensions for field data") + self.data[field] = field_data + + def get(self): + """Convenience function for accesssing box coordinates. + + Returns: + a numpy array of shape [N, 4] representing box corners + """ + return self.get_field("boxes") + + def get_field(self, field): + """Accesses data associated with the specified field in the box collection. + + Args: + field: a string parameter used to speficy a related field to be accessed. + + Returns: + a numpy 1-d array representing data of an associated field + + Raises: + ValueError: if invalid field + """ + if not self.has_field(field): + raise ValueError("field {} does not exist".format(field)) + return self.data[field] + + def get_coordinates(self): + """Get corner coordinates of boxes. + + Returns: + a list of 4 1-d numpy arrays [y_min, x_min, y_max, x_max] + """ + box_coordinates = self.get() + y_min = box_coordinates[:, 0] + x_min = box_coordinates[:, 1] + y_max = box_coordinates[:, 2] + x_max = box_coordinates[:, 3] + return [y_min, x_min, y_max, x_max] + + def _is_valid_boxes(self, data): + """Check whether data fullfills the format of N*[ymin, xmin, ymax, xmin]. + + Args: + data: a numpy array of shape [N, 4] representing box coordinates + + Returns: + a boolean indicating whether all ymax of boxes are equal or greater than + ymin, and all xmax of boxes are equal or greater than xmin. + """ + if data.shape[0] > 0: + for i in range(data.shape[0]): + if data[i, 0] > data[i, 2] or data[i, 1] > data[i, 3]: + return False + return True diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_list_ops.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_list_ops.py" new file mode 100644 index 0000000..7921cff --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_list_ops.py" @@ -0,0 +1,593 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Bounding Box List operations for Numpy BoxLists. + +Example box operations that are supported: + * Areas: compute bounding box areas + * IOU: pairwise intersection-over-union scores +""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) +import numpy as np + +from . import np_box_list, np_box_ops + + +class SortOrder(object): + """Enum class for sort order. + + Attributes: + ascend: ascend order. + descend: descend order. + """ + + ASCEND = 1 + DESCEND = 2 + + +def area(boxlist): + """Computes area of boxes. + + Args: + boxlist: BoxList holding N boxes + + Returns: + a numpy array with shape [N*1] representing box areas + """ + y_min, x_min, y_max, x_max = boxlist.get_coordinates() + return (y_max - y_min) * (x_max - x_min) + + +def intersection(boxlist1, boxlist2): + """Compute pairwise intersection areas between boxes. + + Args: + boxlist1: BoxList holding N boxes + boxlist2: BoxList holding M boxes + + Returns: + a numpy array with shape [N*M] representing pairwise intersection area + """ + return np_box_ops.intersection(boxlist1.get(), boxlist2.get()) + + +def iou(boxlist1, boxlist2): + """Computes pairwise intersection-over-union between box collections. + + Args: + boxlist1: BoxList holding N boxes + boxlist2: BoxList holding M boxes + + Returns: + a numpy array with shape [N, M] representing pairwise iou scores. + """ + return np_box_ops.iou(boxlist1.get(), boxlist2.get()) + + +def ioa(boxlist1, boxlist2): + """Computes pairwise intersection-over-area between box collections. + + Intersection-over-area (ioa) between two boxes box1 and box2 is defined as + their intersection area over box2's area. Note that ioa is not symmetric, + that is, IOA(box1, box2) != IOA(box2, box1). + + Args: + boxlist1: BoxList holding N boxes + boxlist2: BoxList holding M boxes + + Returns: + a numpy array with shape [N, M] representing pairwise ioa scores. + """ + return np_box_ops.ioa(boxlist1.get(), boxlist2.get()) + + +def gather(boxlist, indices, fields=None): + """Gather boxes from BoxList according to indices and return new BoxList. + + By default, gather returns boxes corresponding to the input index list, as + well as all additional fields stored in the boxlist (indexing into the + first dimension). However one can optionally only gather from a + subset of fields. + + Args: + boxlist: BoxList holding N boxes + indices: a 1-d numpy array of type int_ + fields: (optional) list of fields to also gather from. If None (default), + all fields are gathered from. Pass an empty fields list to only gather + the box coordinates. + + Returns: + subboxlist: a BoxList corresponding to the subset of the input BoxList + specified by indices + + Raises: + ValueError: if specified field is not contained in boxlist or if the + indices are not of type int_ + """ + if indices.size: + if np.amax(indices) >= boxlist.num_boxes() or np.amin(indices) < 0: + raise ValueError("indices are out of valid range.") + subboxlist = np_box_list.BoxList(boxlist.get()[indices, :]) + if fields is None: + fields = boxlist.get_extra_fields() + for field in fields: + extra_field_data = boxlist.get_field(field) + subboxlist.add_field(field, extra_field_data[indices, ...]) + return subboxlist + + +def sort_by_field(boxlist, field, order=SortOrder.DESCEND): + """Sort boxes and associated fields according to a scalar field. + + A common use case is reordering the boxes according to descending scores. + + Args: + boxlist: BoxList holding N boxes. + field: A BoxList field for sorting and reordering the BoxList. + order: (Optional) 'descend' or 'ascend'. Default is descend. + + Returns: + sorted_boxlist: A sorted BoxList with the field in the specified order. + + Raises: + ValueError: if specified field does not exist or is not of single dimension. + ValueError: if the order is not either descend or ascend. + """ + if not boxlist.has_field(field): + raise ValueError("Field " + field + " does not exist") + if len(boxlist.get_field(field).shape) != 1: + raise ValueError("Field " + field + "should be single dimension.") + if order != SortOrder.DESCEND and order != SortOrder.ASCEND: + raise ValueError("Invalid sort order") + + field_to_sort = boxlist.get_field(field) + sorted_indices = np.argsort(field_to_sort) + if order == SortOrder.DESCEND: + sorted_indices = sorted_indices[::-1] + return gather(boxlist, sorted_indices) + + +def non_max_suppression( + boxlist, max_output_size=10000, iou_threshold=1.0, score_threshold=-10.0 +): + """Non maximum suppression. + + This op greedily selects a subset of detection bounding boxes, pruning + away boxes that have high IOU (intersection over union) overlap (> thresh) + with already selected boxes. In each iteration, the detected bounding box with + highest score in the available pool is selected. + + Args: + boxlist: BoxList holding N boxes. Must contain a 'scores' field + representing detection scores. All scores belong to the same class. + max_output_size: maximum number of retained boxes + iou_threshold: intersection over union threshold. + score_threshold: minimum score threshold. Remove the boxes with scores + less than this value. Default value is set to -10. A very + low threshold to pass pretty much all the boxes, unless + the user sets a different score threshold. + + Returns: + a BoxList holding M boxes where M <= max_output_size + Raises: + ValueError: if 'scores' field does not exist + ValueError: if threshold is not in [0, 1] + ValueError: if max_output_size < 0 + """ + if not boxlist.has_field("scores"): + raise ValueError("Field scores does not exist") + if iou_threshold < 0.0 or iou_threshold > 1.0: + raise ValueError("IOU threshold must be in [0, 1]") + if max_output_size < 0: + raise ValueError("max_output_size must be bigger than 0.") + + boxlist = filter_scores_greater_than(boxlist, score_threshold) + if boxlist.num_boxes() == 0: + return boxlist + + boxlist = sort_by_field(boxlist, "scores") + + # Prevent further computation if NMS is disabled. + if iou_threshold == 1.0: + if boxlist.num_boxes() > max_output_size: + selected_indices = np.arange(max_output_size) + return gather(boxlist, selected_indices) + else: + return boxlist + + boxes = boxlist.get() + num_boxes = boxlist.num_boxes() + # is_index_valid is True only for all remaining valid boxes, + is_index_valid = np.full(num_boxes, 1, dtype=bool) + selected_indices = [] + num_output = 0 + for i in range(num_boxes): + if num_output < max_output_size: + if is_index_valid[i]: + num_output += 1 + selected_indices.append(i) + is_index_valid[i] = False + valid_indices = np.where(is_index_valid)[0] + if valid_indices.size == 0: + break + + intersect_over_union = np_box_ops.iou( + np.expand_dims(boxes[i, :], axis=0), boxes[valid_indices, :] + ) + intersect_over_union = np.squeeze(intersect_over_union, axis=0) + is_index_valid[valid_indices] = np.logical_and( + is_index_valid[valid_indices], + intersect_over_union <= iou_threshold, + ) + return gather(boxlist, np.array(selected_indices)) + + +def multi_class_non_max_suppression( + boxlist, score_thresh, iou_thresh, max_output_size +): + """Multi-class version of non maximum suppression. + + This op greedily selects a subset of detection bounding boxes, pruning + away boxes that have high IOU (intersection over union) overlap (> thresh) + with already selected boxes. It operates independently for each class for + which scores are provided (via the scores field of the input box_list), + pruning boxes with score less than a provided threshold prior to + applying NMS. + + Args: + boxlist: BoxList holding N boxes. Must contain a 'scores' field + representing detection scores. This scores field is a tensor that can + be 1 dimensional (in the case of a single class) or 2-dimensional, which + which case we assume that it takes the shape [num_boxes, num_classes]. + We further assume that this rank is known statically and that + scores.shape[1] is also known (i.e., the number of classes is fixed + and known at graph construction time). + score_thresh: scalar threshold for score (low scoring boxes are removed). + iou_thresh: scalar threshold for IOU (boxes that that high IOU overlap + with previously selected boxes are removed). + max_output_size: maximum number of retained boxes per class. + + Returns: + a BoxList holding M boxes with a rank-1 scores field representing + corresponding scores for each box with scores sorted in decreasing order + and a rank-1 classes field representing a class label for each box. + Raises: + ValueError: if iou_thresh is not in [0, 1] or if input boxlist does not have + a valid scores field. + """ + if not 0 <= iou_thresh <= 1.0: + raise ValueError("thresh must be between 0 and 1") + if not isinstance(boxlist, np_box_list.BoxList): + raise ValueError("boxlist must be a BoxList") + if not boxlist.has_field("scores"): + raise ValueError("input boxlist must have 'scores' field") + scores = boxlist.get_field("scores") + if len(scores.shape) == 1: + scores = np.reshape(scores, [-1, 1]) + elif len(scores.shape) == 2: + if scores.shape[1] is None: + raise ValueError( + "scores field must have statically defined second " "dimension" + ) + else: + raise ValueError("scores field must be of rank 1 or 2") + num_boxes = boxlist.num_boxes() + num_scores = scores.shape[0] + num_classes = scores.shape[1] + + if num_boxes != num_scores: + raise ValueError("Incorrect scores field length: actual vs expected.") + + selected_boxes_list = [] + for class_idx in range(num_classes): + boxlist_and_class_scores = np_box_list.BoxList(boxlist.get()) + class_scores = np.reshape(scores[0:num_scores, class_idx], [-1]) + boxlist_and_class_scores.add_field("scores", class_scores) + boxlist_filt = filter_scores_greater_than( + boxlist_and_class_scores, score_thresh + ) + nms_result = non_max_suppression( + boxlist_filt, + max_output_size=max_output_size, + iou_threshold=iou_thresh, + score_threshold=score_thresh, + ) + nms_result.add_field( + "classes", np.zeros_like(nms_result.get_field("scores")) + class_idx + ) + selected_boxes_list.append(nms_result) + selected_boxes = concatenate(selected_boxes_list) + sorted_boxes = sort_by_field(selected_boxes, "scores") + return sorted_boxes + + +def scale(boxlist, y_scale, x_scale): + """Scale box coordinates in x and y dimensions. + + Args: + boxlist: BoxList holding N boxes + y_scale: float + x_scale: float + + Returns: + boxlist: BoxList holding N boxes + """ + y_min, x_min, y_max, x_max = np.array_split(boxlist.get(), 4, axis=1) + y_min = y_scale * y_min + y_max = y_scale * y_max + x_min = x_scale * x_min + x_max = x_scale * x_max + scaled_boxlist = np_box_list.BoxList( + np.hstack([y_min, x_min, y_max, x_max]) + ) + + fields = boxlist.get_extra_fields() + for field in fields: + extra_field_data = boxlist.get_field(field) + scaled_boxlist.add_field(field, extra_field_data) + + return scaled_boxlist + + +def clip_to_window(boxlist, window): + """Clip bounding boxes to a window. + + This op clips input bounding boxes (represented by bounding box + corners) to a window, optionally filtering out boxes that do not + overlap at all with the window. + + Args: + boxlist: BoxList holding M_in boxes + window: a numpy array of shape [4] representing the + [y_min, x_min, y_max, x_max] window to which the op + should clip boxes. + + Returns: + a BoxList holding M_out boxes where M_out <= M_in + """ + y_min, x_min, y_max, x_max = np.array_split(boxlist.get(), 4, axis=1) + win_y_min = window[0] + win_x_min = window[1] + win_y_max = window[2] + win_x_max = window[3] + y_min_clipped = np.fmax(np.fmin(y_min, win_y_max), win_y_min) + y_max_clipped = np.fmax(np.fmin(y_max, win_y_max), win_y_min) + x_min_clipped = np.fmax(np.fmin(x_min, win_x_max), win_x_min) + x_max_clipped = np.fmax(np.fmin(x_max, win_x_max), win_x_min) + clipped = np_box_list.BoxList( + np.hstack([y_min_clipped, x_min_clipped, y_max_clipped, x_max_clipped]) + ) + clipped = _copy_extra_fields(clipped, boxlist) + areas = area(clipped) + nonzero_area_indices = np.reshape( + np.nonzero(np.greater(areas, 0.0)), [-1] + ).astype(np.int32) + return gather(clipped, nonzero_area_indices) + + +def prune_non_overlapping_boxes(boxlist1, boxlist2, minoverlap=0.0): + """Prunes the boxes in boxlist1 that overlap less than thresh with boxlist2. + + For each box in boxlist1, we want its IOA to be more than minoverlap with + at least one of the boxes in boxlist2. If it does not, we remove it. + + Args: + boxlist1: BoxList holding N boxes. + boxlist2: BoxList holding M boxes. + minoverlap: Minimum required overlap between boxes, to count them as + overlapping. + + Returns: + A pruned boxlist with size [N', 4]. + """ + intersection_over_area = ioa(boxlist2, boxlist1) # [M, N] tensor + intersection_over_area = np.amax( + intersection_over_area, axis=0 + ) # [N] tensor + keep_bool = np.greater_equal(intersection_over_area, np.array(minoverlap)) + keep_inds = np.nonzero(keep_bool)[0] + new_boxlist1 = gather(boxlist1, keep_inds) + return new_boxlist1 + + +def prune_outside_window(boxlist, window): + """Prunes bounding boxes that fall outside a given window. + + This function prunes bounding boxes that even partially fall outside the given + window. See also ClipToWindow which only prunes bounding boxes that fall + completely outside the window, and clips any bounding boxes that partially + overflow. + + Args: + boxlist: a BoxList holding M_in boxes. + window: a numpy array of size 4, representing [ymin, xmin, ymax, xmax] + of the window. + + Returns: + pruned_corners: a tensor with shape [M_out, 4] where M_out <= M_in. + valid_indices: a tensor with shape [M_out] indexing the valid bounding boxes + in the input tensor. + """ + + y_min, x_min, y_max, x_max = np.array_split(boxlist.get(), 4, axis=1) + win_y_min = window[0] + win_x_min = window[1] + win_y_max = window[2] + win_x_max = window[3] + coordinate_violations = np.hstack( + [ + np.less(y_min, win_y_min), + np.less(x_min, win_x_min), + np.greater(y_max, win_y_max), + np.greater(x_max, win_x_max), + ] + ) + valid_indices = np.reshape( + np.where(np.logical_not(np.max(coordinate_violations, axis=1))), [-1] + ) + return gather(boxlist, valid_indices), valid_indices + + +def concatenate(boxlists, fields=None): + """Concatenate list of BoxLists. + + This op concatenates a list of input BoxLists into a larger BoxList. It also + handles concatenation of BoxList fields as long as the field tensor shapes + are equal except for the first dimension. + + Args: + boxlists: list of BoxList objects + fields: optional list of fields to also concatenate. By default, all + fields from the first BoxList in the list are included in the + concatenation. + + Returns: + a BoxList with number of boxes equal to + sum([boxlist.num_boxes() for boxlist in BoxList]) + Raises: + ValueError: if boxlists is invalid (i.e., is not a list, is empty, or + contains non BoxList objects), or if requested fields are not contained in + all boxlists + """ + if not isinstance(boxlists, list): + raise ValueError("boxlists should be a list") + if not boxlists: + raise ValueError("boxlists should have nonzero length") + for boxlist in boxlists: + if not isinstance(boxlist, np_box_list.BoxList): + raise ValueError( + "all elements of boxlists should be BoxList objects" + ) + concatenated = np_box_list.BoxList( + np.vstack([boxlist.get() for boxlist in boxlists]) + ) + if fields is None: + fields = boxlists[0].get_extra_fields() + for field in fields: + first_field_shape = boxlists[0].get_field(field).shape + first_field_shape = first_field_shape[1:] + for boxlist in boxlists: + if not boxlist.has_field(field): + raise ValueError("boxlist must contain all requested fields") + field_shape = boxlist.get_field(field).shape + field_shape = field_shape[1:] + if field_shape != first_field_shape: + raise ValueError( + "field %s must have same shape for all boxlists " + "except for the 0th dimension." % field + ) + concatenated_field = np.concatenate( + [boxlist.get_field(field) for boxlist in boxlists], axis=0 + ) + concatenated.add_field(field, concatenated_field) + return concatenated + + +def filter_scores_greater_than(boxlist, thresh): + """Filter to keep only boxes with score exceeding a given threshold. + + This op keeps the collection of boxes whose corresponding scores are + greater than the input threshold. + + Args: + boxlist: BoxList holding N boxes. Must contain a 'scores' field + representing detection scores. + thresh: scalar threshold + + Returns: + a BoxList holding M boxes where M <= N + + Raises: + ValueError: if boxlist not a BoxList object or if it does not + have a scores field + """ + if not isinstance(boxlist, np_box_list.BoxList): + raise ValueError("boxlist must be a BoxList") + if not boxlist.has_field("scores"): + raise ValueError("input boxlist must have 'scores' field") + scores = boxlist.get_field("scores") + if len(scores.shape) > 2: + raise ValueError("Scores should have rank 1 or 2") + if len(scores.shape) == 2 and scores.shape[1] != 1: + raise ValueError( + "Scores should have rank 1 or have shape " + "consistent with [None, 1]" + ) + high_score_indices = np.reshape( + np.where(np.greater(scores, thresh)), [-1] + ).astype(np.int32) + return gather(boxlist, high_score_indices) + + +def change_coordinate_frame(boxlist, window): + """Change coordinate frame of the boxlist to be relative to window's frame. + + Given a window of the form [ymin, xmin, ymax, xmax], + changes bounding box coordinates from boxlist to be relative to this window + (e.g., the min corner maps to (0,0) and the max corner maps to (1,1)). + + An example use case is data augmentation: where we are given groundtruth + boxes (boxlist) and would like to randomly crop the image to some + window (window). In this case we need to change the coordinate frame of + each groundtruth box to be relative to this new window. + + Args: + boxlist: A BoxList object holding N boxes. + window: a size 4 1-D numpy array. + + Returns: + Returns a BoxList object with N boxes. + """ + win_height = window[2] - window[0] + win_width = window[3] - window[1] + boxlist_new = scale( + np_box_list.BoxList( + boxlist.get() - [window[0], window[1], window[0], window[1]] + ), + 1.0 / win_height, + 1.0 / win_width, + ) + _copy_extra_fields(boxlist_new, boxlist) + + return boxlist_new + + +def _copy_extra_fields(boxlist_to_copy_to, boxlist_to_copy_from): + """Copies the extra fields of boxlist_to_copy_from to boxlist_to_copy_to. + + Args: + boxlist_to_copy_to: BoxList to which extra fields are copied. + boxlist_to_copy_from: BoxList from which fields are copied. + + Returns: + boxlist_to_copy_to with extra fields. + """ + for field in boxlist_to_copy_from.get_extra_fields(): + boxlist_to_copy_to.add_field( + field, boxlist_to_copy_from.get_field(field) + ) + return boxlist_to_copy_to + + +def _update_valid_indices_by_removing_high_iou_boxes( + selected_indices, is_index_valid, intersect_over_union, threshold +): + max_iou = np.max(intersect_over_union[:, selected_indices], axis=1) + return np.logical_and(is_index_valid, max_iou <= threshold) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_mask_list.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_mask_list.py" new file mode 100644 index 0000000..7fd6b34 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_mask_list.py" @@ -0,0 +1,73 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Numpy BoxMaskList classes and functions.""" + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) +import numpy as np + +from . import np_box_list + + +class BoxMaskList(np_box_list.BoxList): + """Convenience wrapper for BoxList with masks. + + BoxMaskList extends the np_box_list.BoxList to contain masks as well. + In particular, its constructor receives both boxes and masks. Note that the + masks correspond to the full image. + """ + + def __init__(self, box_data, mask_data): + """Constructs box collection. + + Args: + box_data: a numpy array of shape [N, 4] representing box coordinates + mask_data: a numpy array of shape [N, height, width] representing masks + with values are in {0,1}. The masks correspond to the full + image. The height and the width will be equal to image height and width. + + Raises: + ValueError: if bbox data is not a numpy array + ValueError: if invalid dimensions for bbox data + ValueError: if mask data is not a numpy array + ValueError: if invalid dimension for mask data + """ + super(BoxMaskList, self).__init__(box_data) + if not isinstance(mask_data, np.ndarray): + raise ValueError("Mask data must be a numpy array.") + if len(mask_data.shape) != 3: + raise ValueError("Invalid dimensions for mask data.") + if mask_data.dtype != np.uint8: + raise ValueError( + "Invalid data type for mask data: uint8 is required." + ) + if mask_data.shape[0] != box_data.shape[0]: + raise ValueError( + "There should be the same number of boxes and masks." + ) + self.data["masks"] = mask_data + + def get_masks(self): + """Convenience function for accessing masks. + + Returns: + a numpy array of shape [N, height, width] representing masks + """ + return self.get_field("masks") diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_mask_list_ops.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_mask_list_ops.py" new file mode 100644 index 0000000..e919284 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_mask_list_ops.py" @@ -0,0 +1,428 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Operations for np_box_mask_list.BoxMaskList. + +Example box operations that are supported: + * Areas: compute bounding box areas + * IOU: pairwise intersection-over-union scores +""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) +import numpy as np + +from . import np_box_list_ops, np_box_mask_list, np_mask_ops + + +def box_list_to_box_mask_list(boxlist): + """Converts a BoxList containing 'masks' into a BoxMaskList. + + Args: + boxlist: An np_box_list.BoxList object. + + Returns: + An np_box_mask_list.BoxMaskList object. + + Raises: + ValueError: If boxlist does not contain `masks` as a field. + """ + if not boxlist.has_field("masks"): + raise ValueError("boxlist does not contain mask field.") + box_mask_list = np_box_mask_list.BoxMaskList( + box_data=boxlist.get(), mask_data=boxlist.get_field("masks") + ) + extra_fields = boxlist.get_extra_fields() + for key in extra_fields: + if key != "masks": + box_mask_list.data[key] = boxlist.get_field(key) + return box_mask_list + + +def area(box_mask_list): + """Computes area of masks. + + Args: + box_mask_list: np_box_mask_list.BoxMaskList holding N boxes and masks + + Returns: + a numpy array with shape [N*1] representing mask areas + """ + return np_mask_ops.area(box_mask_list.get_masks()) + + +def intersection(box_mask_list1, box_mask_list2): + """Compute pairwise intersection areas between masks. + + Args: + box_mask_list1: BoxMaskList holding N boxes and masks + box_mask_list2: BoxMaskList holding M boxes and masks + + Returns: + a numpy array with shape [N*M] representing pairwise intersection area + """ + return np_mask_ops.intersection( + box_mask_list1.get_masks(), box_mask_list2.get_masks() + ) + + +def iou(box_mask_list1, box_mask_list2): + """Computes pairwise intersection-over-union between box and mask collections. + + Args: + box_mask_list1: BoxMaskList holding N boxes and masks + box_mask_list2: BoxMaskList holding M boxes and masks + + Returns: + a numpy array with shape [N, M] representing pairwise iou scores. + """ + return np_mask_ops.iou( + box_mask_list1.get_masks(), box_mask_list2.get_masks() + ) + + +def ioa(box_mask_list1, box_mask_list2): + """Computes pairwise intersection-over-area between box and mask collections. + + Intersection-over-area (ioa) between two masks mask1 and mask2 is defined as + their intersection area over mask2's area. Note that ioa is not symmetric, + that is, IOA(mask1, mask2) != IOA(mask2, mask1). + + Args: + box_mask_list1: np_box_mask_list.BoxMaskList holding N boxes and masks + box_mask_list2: np_box_mask_list.BoxMaskList holding M boxes and masks + + Returns: + a numpy array with shape [N, M] representing pairwise ioa scores. + """ + return np_mask_ops.ioa( + box_mask_list1.get_masks(), box_mask_list2.get_masks() + ) + + +def gather(box_mask_list, indices, fields=None): + """Gather boxes from np_box_mask_list.BoxMaskList according to indices. + + By default, gather returns boxes corresponding to the input index list, as + well as all additional fields stored in the box_mask_list (indexing into the + first dimension). However one can optionally only gather from a + subset of fields. + + Args: + box_mask_list: np_box_mask_list.BoxMaskList holding N boxes + indices: a 1-d numpy array of type int_ + fields: (optional) list of fields to also gather from. If None (default), + all fields are gathered from. Pass an empty fields list to only gather + the box coordinates. + + Returns: + subbox_mask_list: a np_box_mask_list.BoxMaskList corresponding to the subset + of the input box_mask_list specified by indices + + Raises: + ValueError: if specified field is not contained in box_mask_list or if the + indices are not of type int_ + """ + if fields is not None: + if "masks" not in fields: + fields.append("masks") + return box_list_to_box_mask_list( + np_box_list_ops.gather( + boxlist=box_mask_list, indices=indices, fields=fields + ) + ) + + +def sort_by_field( + box_mask_list, field, order=np_box_list_ops.SortOrder.DESCEND +): + """Sort boxes and associated fields according to a scalar field. + + A common use case is reordering the boxes according to descending scores. + + Args: + box_mask_list: BoxMaskList holding N boxes. + field: A BoxMaskList field for sorting and reordering the BoxMaskList. + order: (Optional) 'descend' or 'ascend'. Default is descend. + + Returns: + sorted_box_mask_list: A sorted BoxMaskList with the field in the specified + order. + """ + return box_list_to_box_mask_list( + np_box_list_ops.sort_by_field( + boxlist=box_mask_list, field=field, order=order + ) + ) + + +def non_max_suppression( + box_mask_list, + max_output_size=10000, + iou_threshold=1.0, + score_threshold=-10.0, +): + """Non maximum suppression. + + This op greedily selects a subset of detection bounding boxes, pruning + away boxes that have high IOU (intersection over union) overlap (> thresh) + with already selected boxes. In each iteration, the detected bounding box with + highest score in the available pool is selected. + + Args: + box_mask_list: np_box_mask_list.BoxMaskList holding N boxes. Must contain + a 'scores' field representing detection scores. All scores belong to the + same class. + max_output_size: maximum number of retained boxes + iou_threshold: intersection over union threshold. + score_threshold: minimum score threshold. Remove the boxes with scores + less than this value. Default value is set to -10. A very + low threshold to pass pretty much all the boxes, unless + the user sets a different score threshold. + + Returns: + an np_box_mask_list.BoxMaskList holding M boxes where M <= max_output_size + + Raises: + ValueError: if 'scores' field does not exist + ValueError: if threshold is not in [0, 1] + ValueError: if max_output_size < 0 + """ + if not box_mask_list.has_field("scores"): + raise ValueError("Field scores does not exist") + if iou_threshold < 0.0 or iou_threshold > 1.0: + raise ValueError("IOU threshold must be in [0, 1]") + if max_output_size < 0: + raise ValueError("max_output_size must be bigger than 0.") + + box_mask_list = filter_scores_greater_than(box_mask_list, score_threshold) + if box_mask_list.num_boxes() == 0: + return box_mask_list + + box_mask_list = sort_by_field(box_mask_list, "scores") + + # Prevent further computation if NMS is disabled. + if iou_threshold == 1.0: + if box_mask_list.num_boxes() > max_output_size: + selected_indices = np.arange(max_output_size) + return gather(box_mask_list, selected_indices) + else: + return box_mask_list + + masks = box_mask_list.get_masks() + num_masks = box_mask_list.num_boxes() + + # is_index_valid is True only for all remaining valid boxes, + is_index_valid = np.full(num_masks, 1, dtype=bool) + selected_indices = [] + num_output = 0 + for i in range(num_masks): + if num_output < max_output_size: + if is_index_valid[i]: + num_output += 1 + selected_indices.append(i) + is_index_valid[i] = False + valid_indices = np.where(is_index_valid)[0] + if valid_indices.size == 0: + break + + intersect_over_union = np_mask_ops.iou( + np.expand_dims(masks[i], axis=0), masks[valid_indices] + ) + intersect_over_union = np.squeeze(intersect_over_union, axis=0) + is_index_valid[valid_indices] = np.logical_and( + is_index_valid[valid_indices], + intersect_over_union <= iou_threshold, + ) + return gather(box_mask_list, np.array(selected_indices)) + + +def multi_class_non_max_suppression( + box_mask_list, score_thresh, iou_thresh, max_output_size +): + """Multi-class version of non maximum suppression. + + This op greedily selects a subset of detection bounding boxes, pruning + away boxes that have high IOU (intersection over union) overlap (> thresh) + with already selected boxes. It operates independently for each class for + which scores are provided (via the scores field of the input box_list), + pruning boxes with score less than a provided threshold prior to + applying NMS. + + Args: + box_mask_list: np_box_mask_list.BoxMaskList holding N boxes. Must contain a + 'scores' field representing detection scores. This scores field is a + tensor that can be 1 dimensional (in the case of a single class) or + 2-dimensional, in which case we assume that it takes the + shape [num_boxes, num_classes]. We further assume that this rank is known + statically and that scores.shape[1] is also known (i.e., the number of + classes is fixed and known at graph construction time). + score_thresh: scalar threshold for score (low scoring boxes are removed). + iou_thresh: scalar threshold for IOU (boxes that that high IOU overlap + with previously selected boxes are removed). + max_output_size: maximum number of retained boxes per class. + + Returns: + a box_mask_list holding M boxes with a rank-1 scores field representing + corresponding scores for each box with scores sorted in decreasing order + and a rank-1 classes field representing a class label for each box. + Raises: + ValueError: if iou_thresh is not in [0, 1] or if input box_mask_list does + not have a valid scores field. + """ + if not 0 <= iou_thresh <= 1.0: + raise ValueError("thresh must be between 0 and 1") + if not isinstance(box_mask_list, np_box_mask_list.BoxMaskList): + raise ValueError("box_mask_list must be a box_mask_list") + if not box_mask_list.has_field("scores"): + raise ValueError("input box_mask_list must have 'scores' field") + scores = box_mask_list.get_field("scores") + if len(scores.shape) == 1: + scores = np.reshape(scores, [-1, 1]) + elif len(scores.shape) == 2: + if scores.shape[1] is None: + raise ValueError( + "scores field must have statically defined second " "dimension" + ) + else: + raise ValueError("scores field must be of rank 1 or 2") + + num_boxes = box_mask_list.num_boxes() + num_scores = scores.shape[0] + num_classes = scores.shape[1] + + if num_boxes != num_scores: + raise ValueError("Incorrect scores field length: actual vs expected.") + + selected_boxes_list = [] + for class_idx in range(num_classes): + box_mask_list_and_class_scores = np_box_mask_list.BoxMaskList( + box_data=box_mask_list.get(), mask_data=box_mask_list.get_masks() + ) + class_scores = np.reshape(scores[0:num_scores, class_idx], [-1]) + box_mask_list_and_class_scores.add_field("scores", class_scores) + box_mask_list_filt = filter_scores_greater_than( + box_mask_list_and_class_scores, score_thresh + ) + nms_result = non_max_suppression( + box_mask_list_filt, + max_output_size=max_output_size, + iou_threshold=iou_thresh, + score_threshold=score_thresh, + ) + nms_result.add_field( + "classes", np.zeros_like(nms_result.get_field("scores")) + class_idx + ) + selected_boxes_list.append(nms_result) + selected_boxes = np_box_list_ops.concatenate(selected_boxes_list) + sorted_boxes = np_box_list_ops.sort_by_field(selected_boxes, "scores") + return box_list_to_box_mask_list(boxlist=sorted_boxes) + + +def prune_non_overlapping_masks(box_mask_list1, box_mask_list2, minoverlap=0.0): + """Prunes the boxes in list1 that overlap less than thresh with list2. + + For each mask in box_mask_list1, we want its IOA to be more than minoverlap + with at least one of the masks in box_mask_list2. If it does not, we remove + it. If the masks are not full size image, we do the pruning based on boxes. + + Args: + box_mask_list1: np_box_mask_list.BoxMaskList holding N boxes and masks. + box_mask_list2: np_box_mask_list.BoxMaskList holding M boxes and masks. + minoverlap: Minimum required overlap between boxes, to count them as + overlapping. + + Returns: + A pruned box_mask_list with size [N', 4]. + """ + intersection_over_area = ioa( + box_mask_list2, box_mask_list1 + ) # [M, N] tensor + intersection_over_area = np.amax( + intersection_over_area, axis=0 + ) # [N] tensor + keep_bool = np.greater_equal(intersection_over_area, np.array(minoverlap)) + keep_inds = np.nonzero(keep_bool)[0] + new_box_mask_list1 = gather(box_mask_list1, keep_inds) + return new_box_mask_list1 + + +def concatenate(box_mask_lists, fields=None): + """Concatenate list of box_mask_lists. + + This op concatenates a list of input box_mask_lists into a larger + box_mask_list. It also + handles concatenation of box_mask_list fields as long as the field tensor + shapes are equal except for the first dimension. + + Args: + box_mask_lists: list of np_box_mask_list.BoxMaskList objects + fields: optional list of fields to also concatenate. By default, all + fields from the first BoxMaskList in the list are included in the + concatenation. + + Returns: + a box_mask_list with number of boxes equal to + sum([box_mask_list.num_boxes() for box_mask_list in box_mask_list]) + Raises: + ValueError: if box_mask_lists is invalid (i.e., is not a list, is empty, or + contains non box_mask_list objects), or if requested fields are not + contained in all box_mask_lists + """ + if fields is not None: + if "masks" not in fields: + fields.append("masks") + return box_list_to_box_mask_list( + np_box_list_ops.concatenate(boxlists=box_mask_lists, fields=fields) + ) + + +def filter_scores_greater_than(box_mask_list, thresh): + """Filter to keep only boxes and masks with score exceeding a given threshold. + + This op keeps the collection of boxes and masks whose corresponding scores are + greater than the input threshold. + + Args: + box_mask_list: BoxMaskList holding N boxes and masks. Must contain a + 'scores' field representing detection scores. + thresh: scalar threshold + + Returns: + a BoxMaskList holding M boxes and masks where M <= N + + Raises: + ValueError: if box_mask_list not a np_box_mask_list.BoxMaskList object or + if it does not have a scores field + """ + if not isinstance(box_mask_list, np_box_mask_list.BoxMaskList): + raise ValueError("box_mask_list must be a BoxMaskList") + if not box_mask_list.has_field("scores"): + raise ValueError("input box_mask_list must have 'scores' field") + scores = box_mask_list.get_field("scores") + if len(scores.shape) > 2: + raise ValueError("Scores should have rank 1 or 2") + if len(scores.shape) == 2 and scores.shape[1] != 1: + raise ValueError( + "Scores should have rank 1 or have shape " + "consistent with [None, 1]" + ) + high_score_indices = np.reshape( + np.where(np.greater(scores, thresh)), [-1] + ).astype(np.int32) + return gather(box_mask_list, high_score_indices) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_ops.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_ops.py" new file mode 100644 index 0000000..7cadc80 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_box_ops.py" @@ -0,0 +1,108 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Operations for [N, 4] numpy arrays representing bounding boxes. + +Example box operations that are supported: + * Areas: compute bounding box areas + * IOU: pairwise intersection-over-union scores +""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) +import numpy as np + + +def area(boxes): + """Computes area of boxes. + + Args: + boxes: Numpy array with shape [N, 4] holding N boxes + + Returns: + a numpy array with shape [N*1] representing box areas + """ + return (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) + + +def intersection(boxes1, boxes2): + """Compute pairwise intersection areas between boxes. + + Args: + boxes1: a numpy array with shape [N, 4] holding N boxes + boxes2: a numpy array with shape [M, 4] holding M boxes + + Returns: + a numpy array with shape [N*M] representing pairwise intersection area + """ + [y_min1, x_min1, y_max1, x_max1] = np.split(boxes1, 4, axis=1) + [y_min2, x_min2, y_max2, x_max2] = np.split(boxes2, 4, axis=1) + + all_pairs_min_ymax = np.minimum(y_max1, np.transpose(y_max2)) + all_pairs_max_ymin = np.maximum(y_min1, np.transpose(y_min2)) + intersect_heights = np.maximum( + np.zeros(all_pairs_max_ymin.shape), + all_pairs_min_ymax - all_pairs_max_ymin, + ) + all_pairs_min_xmax = np.minimum(x_max1, np.transpose(x_max2)) + all_pairs_max_xmin = np.maximum(x_min1, np.transpose(x_min2)) + intersect_widths = np.maximum( + np.zeros(all_pairs_max_xmin.shape), + all_pairs_min_xmax - all_pairs_max_xmin, + ) + return intersect_heights * intersect_widths + + +def iou(boxes1, boxes2): + """Computes pairwise intersection-over-union between box collections. + + Args: + boxes1: a numpy array with shape [N, 4] holding N boxes. + boxes2: a numpy array with shape [M, 4] holding N boxes. + + Returns: + a numpy array with shape [N, M] representing pairwise iou scores. + """ + intersect = intersection(boxes1, boxes2) + area1 = area(boxes1) + area2 = area(boxes2) + union = ( + np.expand_dims(area1, axis=1) + + np.expand_dims(area2, axis=0) + - intersect + ) + return intersect / union + + +def ioa(boxes1, boxes2): + """Computes pairwise intersection-over-area between box collections. + + Intersection-over-area (ioa) between two boxes box1 and box2 is defined as + their intersection area over box2's area. Note that ioa is not symmetric, + that is, IOA(box1, box2) != IOA(box2, box1). + + Args: + boxes1: a numpy array with shape [N, 4] holding N boxes. + boxes2: a numpy array with shape [M, 4] holding N boxes. + + Returns: + a numpy array with shape [N, M] representing pairwise ioa scores. + """ + intersect = intersection(boxes1, boxes2) + areas = np.expand_dims(area(boxes2), axis=0) + return intersect / areas diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_mask_ops.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_mask_ops.py" new file mode 100644 index 0000000..64e36b5 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/np_mask_ops.py" @@ -0,0 +1,130 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Operations for [N, height, width] numpy arrays representing masks. + +Example mask operations that are supported: + * Areas: compute mask areas + * IOU: pairwise intersection-over-union scores +""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) +import numpy as np + +EPSILON = 1e-7 + + +def area(masks): + """Computes area of masks. + + Args: + masks: Numpy array with shape [N, height, width] holding N masks. Masks + values are of type np.uint8 and values are in {0,1}. + + Returns: + a numpy array with shape [N*1] representing mask areas. + + Raises: + ValueError: If masks.dtype is not np.uint8 + """ + if masks.dtype != np.uint8: + raise ValueError("Masks type should be np.uint8") + return np.sum(masks, axis=(1, 2), dtype=np.float32) + + +def intersection(masks1, masks2): + """Compute pairwise intersection areas between masks. + + Args: + masks1: a numpy array with shape [N, height, width] holding N masks. Masks + values are of type np.uint8 and values are in {0,1}. + masks2: a numpy array with shape [M, height, width] holding M masks. Masks + values are of type np.uint8 and values are in {0,1}. + + Returns: + a numpy array with shape [N*M] representing pairwise intersection area. + + Raises: + ValueError: If masks1 and masks2 are not of type np.uint8. + """ + if masks1.dtype != np.uint8 or masks2.dtype != np.uint8: + raise ValueError("masks1 and masks2 should be of type np.uint8") + n = masks1.shape[0] + m = masks2.shape[0] + answer = np.zeros([n, m], dtype=np.float32) + for i in np.arange(n): + for j in np.arange(m): + answer[i, j] = np.sum( + np.minimum(masks1[i], masks2[j]), dtype=np.float32 + ) + return answer + + +def iou(masks1, masks2): + """Computes pairwise intersection-over-union between mask collections. + + Args: + masks1: a numpy array with shape [N, height, width] holding N masks. Masks + values are of type np.uint8 and values are in {0,1}. + masks2: a numpy array with shape [M, height, width] holding N masks. Masks + values are of type np.uint8 and values are in {0,1}. + + Returns: + a numpy array with shape [N, M] representing pairwise iou scores. + + Raises: + ValueError: If masks1 and masks2 are not of type np.uint8. + """ + if masks1.dtype != np.uint8 or masks2.dtype != np.uint8: + raise ValueError("masks1 and masks2 should be of type np.uint8") + intersect = intersection(masks1, masks2) + area1 = area(masks1) + area2 = area(masks2) + union = ( + np.expand_dims(area1, axis=1) + + np.expand_dims(area2, axis=0) + - intersect + ) + return intersect / np.maximum(union, EPSILON) + + +def ioa(masks1, masks2): + """Computes pairwise intersection-over-area between box collections. + + Intersection-over-area (ioa) between two masks, mask1 and mask2 is defined as + their intersection area over mask2's area. Note that ioa is not symmetric, + that is, IOA(mask1, mask2) != IOA(mask2, mask1). + + Args: + masks1: a numpy array with shape [N, height, width] holding N masks. Masks + values are of type np.uint8 and values are in {0,1}. + masks2: a numpy array with shape [M, height, width] holding N masks. Masks + values are of type np.uint8 and values are in {0,1}. + + Returns: + a numpy array with shape [N, M] representing pairwise ioa scores. + + Raises: + ValueError: If masks1 and masks2 are not of type np.uint8. + """ + if masks1.dtype != np.uint8 or masks2.dtype != np.uint8: + raise ValueError("masks1 and masks2 should be of type np.uint8") + intersect = intersection(masks1, masks2) + areas = np.expand_dims(area(masks2), axis=0) + return intersect / (areas + EPSILON) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/object_detection_evaluation.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/object_detection_evaluation.py" new file mode 100644 index 0000000..4810572 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/object_detection_evaluation.py" @@ -0,0 +1,832 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""object_detection_evaluation module. + +ObjectDetectionEvaluation is a class which manages ground truth information of a +object detection dataset, and computes frequently used detection metrics such as +Precision, Recall, CorLoc of the provided detection results. +It supports the following operations: +1) Add ground truth information of images sequentially. +2) Add detection result of images sequentially. +3) Evaluate detection metrics on already inserted detection results. +4) Write evaluation result into a pickle file for future processing or + visualization. + +Note: This module operates on numpy boxes and box lists. +""" + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) +import collections +import logging +import numpy as np +from abc import ABCMeta, abstractmethod + +from . import label_map_util, metrics, per_image_evaluation, standard_fields + + +class DetectionEvaluator(object): + """Interface for object detection evalution classes. + + Example usage of the Evaluator: + ------------------------------ + evaluator = DetectionEvaluator(categories) + + # Detections and groundtruth for image 1. + evaluator.add_single_groundtruth_image_info(...) + evaluator.add_single_detected_image_info(...) + + # Detections and groundtruth for image 2. + evaluator.add_single_groundtruth_image_info(...) + evaluator.add_single_detected_image_info(...) + + metrics_dict = evaluator.evaluate() + """ + + __metaclass__ = ABCMeta + + def __init__(self, categories): + """Constructor. + + Args: + categories: A list of dicts, each of which has the following keys - + 'id': (required) an integer id uniquely identifying this category. + 'name': (required) string representing category name e.g., 'cat', 'dog'. + """ + self._categories = categories + + @abstractmethod + def add_single_ground_truth_image_info(self, image_id, groundtruth_dict): + """Adds groundtruth for a single image to be used for evaluation. + + Args: + image_id: A unique string/integer identifier for the image. + groundtruth_dict: A dictionary of groundtruth numpy arrays required + for evaluations. + """ + pass + + @abstractmethod + def add_single_detected_image_info(self, image_id, detections_dict): + """Adds detections for a single image to be used for evaluation. + + Args: + image_id: A unique string/integer identifier for the image. + detections_dict: A dictionary of detection numpy arrays required + for evaluation. + """ + pass + + @abstractmethod + def evaluate(self): + """Evaluates detections and returns a dictionary of metrics.""" + pass + + @abstractmethod + def clear(self): + """Clears the state to prepare for a fresh evaluation.""" + pass + + +class ObjectDetectionEvaluator(DetectionEvaluator): + """A class to evaluate detections.""" + + def __init__( + self, + categories, + matching_iou_threshold=0.5, + evaluate_corlocs=False, + metric_prefix=None, + use_weighted_mean_ap=False, + evaluate_masks=False, + ): + """Constructor. + + Args: + categories: A list of dicts, each of which has the following keys - + 'id': (required) an integer id uniquely identifying this category. + 'name': (required) string representing category name e.g., 'cat', 'dog'. + matching_iou_threshold: IOU threshold to use for matching groundtruth + boxes to detection boxes. + evaluate_corlocs: (optional) boolean which determines if corloc scores + are to be returned or not. + metric_prefix: (optional) string prefix for metric name; if None, no + prefix is used. + use_weighted_mean_ap: (optional) boolean which determines if the mean + average precision is computed directly from the scores and tp_fp_labels + of all classes. + evaluate_masks: If False, evaluation will be performed based on boxes. + If True, mask evaluation will be performed instead. + + Raises: + ValueError: If the category ids are not 1-indexed. + """ + super(ObjectDetectionEvaluator, self).__init__(categories) + self._num_classes = max([cat["id"] for cat in categories]) + if min(cat["id"] for cat in categories) < 1: + raise ValueError("Classes should be 1-indexed.") + self._matching_iou_threshold = matching_iou_threshold + self._use_weighted_mean_ap = use_weighted_mean_ap + self._label_id_offset = 1 + self._evaluate_masks = evaluate_masks + self._evaluation = ObjectDetectionEvaluation( + num_groundtruth_classes=self._num_classes, + matching_iou_threshold=self._matching_iou_threshold, + use_weighted_mean_ap=self._use_weighted_mean_ap, + label_id_offset=self._label_id_offset, + ) + self._image_ids = set([]) + self._evaluate_corlocs = evaluate_corlocs + self._metric_prefix = (metric_prefix + "_") if metric_prefix else "" + + def add_single_ground_truth_image_info(self, image_id, groundtruth_dict): + """Adds groundtruth for a single image to be used for evaluation. + + Args: + image_id: A unique string/integer identifier for the image. + groundtruth_dict: A dictionary containing - + standard_fields.InputDataFields.groundtruth_boxes: float32 numpy array + of shape [num_boxes, 4] containing `num_boxes` groundtruth boxes of + the format [ymin, xmin, ymax, xmax] in absolute image coordinates. + standard_fields.InputDataFields.groundtruth_classes: integer numpy array + of shape [num_boxes] containing 1-indexed groundtruth classes for the + boxes. + standard_fields.InputDataFields.groundtruth_difficult: Optional length + M numpy boolean array denoting whether a ground truth box is a + difficult instance or not. This field is optional to support the case + that no boxes are difficult. + standard_fields.InputDataFields.groundtruth_instance_masks: Optional + numpy array of shape [num_boxes, height, width] with values in {0, 1}. + + Raises: + ValueError: On adding groundtruth for an image more than once. Will also + raise error if instance masks are not in groundtruth dictionary. + """ + if image_id in self._image_ids: + raise ValueError("Image with id {} already added.".format(image_id)) + + groundtruth_classes = ( + groundtruth_dict[ + standard_fields.InputDataFields.groundtruth_classes + ] + - self._label_id_offset + ) + # If the key is not present in the groundtruth_dict or the array is empty + # (unless there are no annotations for the groundtruth on this image) + # use values from the dictionary or insert None otherwise. + if ( + standard_fields.InputDataFields.groundtruth_difficult + in groundtruth_dict.keys() + and ( + groundtruth_dict[ + standard_fields.InputDataFields.groundtruth_difficult + ].size + or not groundtruth_classes.size + ) + ): + groundtruth_difficult = groundtruth_dict[ + standard_fields.InputDataFields.groundtruth_difficult + ] + else: + groundtruth_difficult = None + if not len(self._image_ids) % 1000: + logging.warning( + "image %s does not have groundtruth difficult flag specified", + image_id, + ) + groundtruth_masks = None + if self._evaluate_masks: + if ( + standard_fields.InputDataFields.groundtruth_instance_masks + not in groundtruth_dict + ): + raise ValueError( + "Instance masks not in groundtruth dictionary." + ) + groundtruth_masks = groundtruth_dict[ + standard_fields.InputDataFields.groundtruth_instance_masks + ] + self._evaluation.add_single_ground_truth_image_info( + image_key=image_id, + groundtruth_boxes=groundtruth_dict[ + standard_fields.InputDataFields.groundtruth_boxes + ], + groundtruth_class_labels=groundtruth_classes, + groundtruth_is_difficult_list=groundtruth_difficult, + groundtruth_masks=groundtruth_masks, + ) + self._image_ids.update([image_id]) + + def add_single_detected_image_info(self, image_id, detections_dict): + """Adds detections for a single image to be used for evaluation. + + Args: + image_id: A unique string/integer identifier for the image. + detections_dict: A dictionary containing - + standard_fields.DetectionResultFields.detection_boxes: float32 numpy + array of shape [num_boxes, 4] containing `num_boxes` detection boxes + of the format [ymin, xmin, ymax, xmax] in absolute image coordinates. + standard_fields.DetectionResultFields.detection_scores: float32 numpy + array of shape [num_boxes] containing detection scores for the boxes. + standard_fields.DetectionResultFields.detection_classes: integer numpy + array of shape [num_boxes] containing 1-indexed detection classes for + the boxes. + standard_fields.DetectionResultFields.detection_masks: uint8 numpy + array of shape [num_boxes, height, width] containing `num_boxes` masks + of values ranging between 0 and 1. + + Raises: + ValueError: If detection masks are not in detections dictionary. + """ + detection_classes = ( + detections_dict[ + standard_fields.DetectionResultFields.detection_classes + ] + - self._label_id_offset + ) + detection_masks = None + if self._evaluate_masks: + if ( + standard_fields.DetectionResultFields.detection_masks + not in detections_dict + ): + raise ValueError( + "Detection masks not in detections dictionary." + ) + detection_masks = detections_dict[ + standard_fields.DetectionResultFields.detection_masks + ] + self._evaluation.add_single_detected_image_info( + image_key=image_id, + detected_boxes=detections_dict[ + standard_fields.DetectionResultFields.detection_boxes + ], + detected_scores=detections_dict[ + standard_fields.DetectionResultFields.detection_scores + ], + detected_class_labels=detection_classes, + detected_masks=detection_masks, + ) + + def evaluate(self): + """Compute evaluation result. + + Returns: + A dictionary of metrics with the following fields - + + 1. summary_metrics: + 'Precision/mAP@IOU': mean average precision at + the specified IOU threshold. + + 2. per_category_ap: category specific results with keys of the form + 'PerformanceByCategory/mAP@IOU/category'. + """ + ( + per_class_ap, + mean_ap, + _, + _, + per_class_corloc, + mean_corloc, + ) = self._evaluation.evaluate() + pascal_metrics = { + self._metric_prefix + + "Precision/mAP@{}IOU".format( + self._matching_iou_threshold + ): mean_ap + } + if self._evaluate_corlocs: + pascal_metrics[ + self._metric_prefix + + "Precision/meanCorLoc@{}IOU".format( + self._matching_iou_threshold + ) + ] = mean_corloc + category_index = label_map_util.create_category_index(self._categories) + for idx in range(per_class_ap.size): + if idx + self._label_id_offset in category_index: + display_name = ( + self._metric_prefix + + "PerformanceByCategory/AP@{}IOU/{}".format( + self._matching_iou_threshold, + category_index[idx + self._label_id_offset]["name"], + ) + ) + pascal_metrics[display_name] = per_class_ap[idx] + + # Optionally add CorLoc metrics.classes + if self._evaluate_corlocs: + display_name = ( + self._metric_prefix + + "PerformanceByCategory/CorLoc@{}IOU/{}".format( + self._matching_iou_threshold, + category_index[idx + self._label_id_offset]["name"], + ) + ) + pascal_metrics[display_name] = per_class_corloc[idx] + + return pascal_metrics + + def clear(self): + """Clears the state to prepare for a fresh evaluation.""" + self._evaluation = ObjectDetectionEvaluation( + num_groundtruth_classes=self._num_classes, + matching_iou_threshold=self._matching_iou_threshold, + use_weighted_mean_ap=self._use_weighted_mean_ap, + label_id_offset=self._label_id_offset, + ) + self._image_ids.clear() + + +class PascalDetectionEvaluator(ObjectDetectionEvaluator): + """A class to evaluate detections using PASCAL metrics.""" + + def __init__(self, categories, matching_iou_threshold=0.5): + super(PascalDetectionEvaluator, self).__init__( + categories, + matching_iou_threshold=matching_iou_threshold, + evaluate_corlocs=False, + metric_prefix="PascalBoxes", + use_weighted_mean_ap=False, + ) + + +class WeightedPascalDetectionEvaluator(ObjectDetectionEvaluator): + """A class to evaluate detections using weighted PASCAL metrics. + + Weighted PASCAL metrics computes the mean average precision as the average + precision given the scores and tp_fp_labels of all classes. In comparison, + PASCAL metrics computes the mean average precision as the mean of the + per-class average precisions. + + This definition is very similar to the mean of the per-class average + precisions weighted by class frequency. However, they are typically not the + same as the average precision is not a linear function of the scores and + tp_fp_labels. + """ + + def __init__(self, categories, matching_iou_threshold=0.5): + super(WeightedPascalDetectionEvaluator, self).__init__( + categories, + matching_iou_threshold=matching_iou_threshold, + evaluate_corlocs=False, + metric_prefix="WeightedPascalBoxes", + use_weighted_mean_ap=True, + ) + + +class PascalInstanceSegmentationEvaluator(ObjectDetectionEvaluator): + """A class to evaluate instance masks using PASCAL metrics.""" + + def __init__(self, categories, matching_iou_threshold=0.5): + super(PascalInstanceSegmentationEvaluator, self).__init__( + categories, + matching_iou_threshold=matching_iou_threshold, + evaluate_corlocs=False, + metric_prefix="PascalMasks", + use_weighted_mean_ap=False, + evaluate_masks=True, + ) + + +class WeightedPascalInstanceSegmentationEvaluator(ObjectDetectionEvaluator): + """A class to evaluate instance masks using weighted PASCAL metrics. + + Weighted PASCAL metrics computes the mean average precision as the average + precision given the scores and tp_fp_labels of all classes. In comparison, + PASCAL metrics computes the mean average precision as the mean of the + per-class average precisions. + + This definition is very similar to the mean of the per-class average + precisions weighted by class frequency. However, they are typically not the + same as the average precision is not a linear function of the scores and + tp_fp_labels. + """ + + def __init__(self, categories, matching_iou_threshold=0.5): + super(WeightedPascalInstanceSegmentationEvaluator, self).__init__( + categories, + matching_iou_threshold=matching_iou_threshold, + evaluate_corlocs=False, + metric_prefix="WeightedPascalMasks", + use_weighted_mean_ap=True, + evaluate_masks=True, + ) + + +class OpenImagesDetectionEvaluator(ObjectDetectionEvaluator): + """A class to evaluate detections using Open Images V2 metrics. + + Open Images V2 introduce group_of type of bounding boxes and this metric + handles those boxes appropriately. + """ + + def __init__( + self, categories, matching_iou_threshold=0.5, evaluate_corlocs=False + ): + """Constructor. + + Args: + categories: A list of dicts, each of which has the following keys - + 'id': (required) an integer id uniquely identifying this category. + 'name': (required) string representing category name e.g., 'cat', 'dog'. + matching_iou_threshold: IOU threshold to use for matching groundtruth + boxes to detection boxes. + evaluate_corlocs: if True, additionally evaluates and returns CorLoc. + """ + super(OpenImagesDetectionEvaluator, self).__init__( + categories, + matching_iou_threshold, + evaluate_corlocs, + metric_prefix="OpenImagesV2", + ) + + def add_single_ground_truth_image_info(self, image_id, groundtruth_dict): + """Adds groundtruth for a single image to be used for evaluation. + + Args: + image_id: A unique string/integer identifier for the image. + groundtruth_dict: A dictionary containing - + standard_fields.InputDataFields.groundtruth_boxes: float32 numpy array + of shape [num_boxes, 4] containing `num_boxes` groundtruth boxes of + the format [ymin, xmin, ymax, xmax] in absolute image coordinates. + standard_fields.InputDataFields.groundtruth_classes: integer numpy array + of shape [num_boxes] containing 1-indexed groundtruth classes for the + boxes. + standard_fields.InputDataFields.groundtruth_group_of: Optional length + M numpy boolean array denoting whether a groundtruth box contains a + group of instances. + + Raises: + ValueError: On adding groundtruth for an image more than once. + """ + if image_id in self._image_ids: + raise ValueError("Image with id {} already added.".format(image_id)) + + groundtruth_classes = ( + groundtruth_dict[ + standard_fields.InputDataFields.groundtruth_classes + ] + - self._label_id_offset + ) + # If the key is not present in the groundtruth_dict or the array is empty + # (unless there are no annotations for the groundtruth on this image) + # use values from the dictionary or insert None otherwise. + if ( + standard_fields.InputDataFields.groundtruth_group_of + in groundtruth_dict.keys() + and ( + groundtruth_dict[ + standard_fields.InputDataFields.groundtruth_group_of + ].size + or not groundtruth_classes.size + ) + ): + groundtruth_group_of = groundtruth_dict[ + standard_fields.InputDataFields.groundtruth_group_of + ] + else: + groundtruth_group_of = None + if not len(self._image_ids) % 1000: + logging.warning( + "image %s does not have groundtruth group_of flag specified", + image_id, + ) + self._evaluation.add_single_ground_truth_image_info( + image_id, + groundtruth_dict[standard_fields.InputDataFields.groundtruth_boxes], + groundtruth_classes, + groundtruth_is_difficult_list=None, + groundtruth_is_group_of_list=groundtruth_group_of, + ) + self._image_ids.update([image_id]) + + +ObjectDetectionEvalMetrics = collections.namedtuple( + "ObjectDetectionEvalMetrics", + [ + "average_precisions", + "mean_ap", + "precisions", + "recalls", + "corlocs", + "mean_corloc", + ], +) + + +class ObjectDetectionEvaluation(object): + """Internal implementation of Pascal object detection metrics.""" + + def __init__( + self, + num_groundtruth_classes, + matching_iou_threshold=0.5, + nms_iou_threshold=1.0, + nms_max_output_boxes=10000, + use_weighted_mean_ap=False, + label_id_offset=0, + ): + if num_groundtruth_classes < 1: + raise ValueError( + "Need at least 1 groundtruth class for evaluation." + ) + + self.per_image_eval = per_image_evaluation.PerImageEvaluation( + num_groundtruth_classes=num_groundtruth_classes, + matching_iou_threshold=matching_iou_threshold, + ) + self.num_class = num_groundtruth_classes + self.use_weighted_mean_ap = use_weighted_mean_ap + self.label_id_offset = label_id_offset + + self.groundtruth_boxes = {} + self.groundtruth_class_labels = {} + self.groundtruth_masks = {} + self.groundtruth_is_difficult_list = {} + self.groundtruth_is_group_of_list = {} + self.num_gt_instances_per_class = np.zeros(self.num_class, dtype=int) + self.num_gt_imgs_per_class = np.zeros(self.num_class, dtype=int) + + self._initialize_detections() + + def _initialize_detections(self): + self.detection_keys = set() + self.scores_per_class = [[] for _ in range(self.num_class)] + self.tp_fp_labels_per_class = [[] for _ in range(self.num_class)] + self.num_images_correctly_detected_per_class = np.zeros(self.num_class) + self.average_precision_per_class = np.empty(self.num_class, dtype=float) + self.average_precision_per_class.fill(np.nan) + self.precisions_per_class = [] + self.recalls_per_class = [] + self.corloc_per_class = np.ones(self.num_class, dtype=float) + + def clear_detections(self): + self._initialize_detections() + + def add_single_ground_truth_image_info( + self, + image_key, + groundtruth_boxes, + groundtruth_class_labels, + groundtruth_is_difficult_list=None, + groundtruth_is_group_of_list=None, + groundtruth_masks=None, + ): + """Adds groundtruth for a single image to be used for evaluation. + + Args: + image_key: A unique string/integer identifier for the image. + groundtruth_boxes: float32 numpy array of shape [num_boxes, 4] + containing `num_boxes` groundtruth boxes of the format + [ymin, xmin, ymax, xmax] in absolute image coordinates. + groundtruth_class_labels: integer numpy array of shape [num_boxes] + containing 0-indexed groundtruth classes for the boxes. + groundtruth_is_difficult_list: A length M numpy boolean array denoting + whether a ground truth box is a difficult instance or not. To support + the case that no boxes are difficult, it is by default set as None. + groundtruth_is_group_of_list: A length M numpy boolean array denoting + whether a ground truth box is a group-of box or not. To support + the case that no boxes are groups-of, it is by default set as None. + groundtruth_masks: uint8 numpy array of shape + [num_boxes, height, width] containing `num_boxes` groundtruth masks. + The mask values range from 0 to 1. + """ + if image_key in self.groundtruth_boxes: + logging.warning( + "image %s has already been added to the ground truth database.", + image_key, + ) + return + + self.groundtruth_boxes[image_key] = groundtruth_boxes + self.groundtruth_class_labels[image_key] = groundtruth_class_labels + self.groundtruth_masks[image_key] = groundtruth_masks + if groundtruth_is_difficult_list is None: + num_boxes = groundtruth_boxes.shape[0] + groundtruth_is_difficult_list = np.zeros(num_boxes, dtype=bool) + self.groundtruth_is_difficult_list[ + image_key + ] = groundtruth_is_difficult_list.astype(dtype=bool) + if groundtruth_is_group_of_list is None: + num_boxes = groundtruth_boxes.shape[0] + groundtruth_is_group_of_list = np.zeros(num_boxes, dtype=bool) + self.groundtruth_is_group_of_list[ + image_key + ] = groundtruth_is_group_of_list.astype(dtype=bool) + + self._update_ground_truth_statistics( + groundtruth_class_labels, + groundtruth_is_difficult_list.astype(dtype=bool), + groundtruth_is_group_of_list.astype(dtype=bool), + ) + + def add_single_detected_image_info( + self, + image_key, + detected_boxes, + detected_scores, + detected_class_labels, + detected_masks=None, + ): + """Adds detections for a single image to be used for evaluation. + + Args: + image_key: A unique string/integer identifier for the image. + detected_boxes: float32 numpy array of shape [num_boxes, 4] + containing `num_boxes` detection boxes of the format + [ymin, xmin, ymax, xmax] in absolute image coordinates. + detected_scores: float32 numpy array of shape [num_boxes] containing + detection scores for the boxes. + detected_class_labels: integer numpy array of shape [num_boxes] containing + 0-indexed detection classes for the boxes. + detected_masks: np.uint8 numpy array of shape [num_boxes, height, width] + containing `num_boxes` detection masks with values ranging + between 0 and 1. + + Raises: + ValueError: if the number of boxes, scores and class labels differ in + length. + """ + if len(detected_boxes) != len(detected_scores) or len( + detected_boxes + ) != len(detected_class_labels): + raise ValueError( + "detected_boxes, detected_scores and " + "detected_class_labels should all have same lengths. Got" + "[%d, %d, %d]" % len(detected_boxes), + len(detected_scores), + len(detected_class_labels), + ) + + if image_key in self.detection_keys: + logging.warning( + "image %s has already been added to the detection result database", + image_key, + ) + return + + self.detection_keys.add(image_key) + if image_key in self.groundtruth_boxes: + groundtruth_boxes = self.groundtruth_boxes[image_key] + groundtruth_class_labels = self.groundtruth_class_labels[image_key] + # Masks are popped instead of look up. The reason is that we do not want + # to keep all masks in memory which can cause memory overflow. + groundtruth_masks = self.groundtruth_masks.pop(image_key) + groundtruth_is_difficult_list = self.groundtruth_is_difficult_list[ + image_key + ] + groundtruth_is_group_of_list = self.groundtruth_is_group_of_list[ + image_key + ] + else: + groundtruth_boxes = np.empty(shape=[0, 4], dtype=float) + groundtruth_class_labels = np.array([], dtype=int) + if detected_masks is None: + groundtruth_masks = None + else: + groundtruth_masks = np.empty(shape=[0, 1, 1], dtype=float) + groundtruth_is_difficult_list = np.array([], dtype=bool) + groundtruth_is_group_of_list = np.array([], dtype=bool) + ( + scores, + tp_fp_labels, + ) = self.per_image_eval.compute_object_detection_metrics( + detected_boxes=detected_boxes, + detected_scores=detected_scores, + detected_class_labels=detected_class_labels, + groundtruth_boxes=groundtruth_boxes, + groundtruth_class_labels=groundtruth_class_labels, + groundtruth_is_difficult_list=groundtruth_is_difficult_list, + groundtruth_is_group_of_list=groundtruth_is_group_of_list, + detected_masks=detected_masks, + groundtruth_masks=groundtruth_masks, + ) + + for i in range(self.num_class): + if scores[i].shape[0] > 0: + self.scores_per_class[i].append(scores[i]) + self.tp_fp_labels_per_class[i].append(tp_fp_labels[i]) + + def _update_ground_truth_statistics( + self, + groundtruth_class_labels, + groundtruth_is_difficult_list, + groundtruth_is_group_of_list, + ): + """Update grouth truth statitistics. + + 1. Difficult boxes are ignored when counting the number of ground truth + instances as done in Pascal VOC devkit. + 2. Difficult boxes are treated as normal boxes when computing CorLoc related + statitistics. + + Args: + groundtruth_class_labels: An integer numpy array of length M, + representing M class labels of object instances in ground truth + groundtruth_is_difficult_list: A boolean numpy array of length M denoting + whether a ground truth box is a difficult instance or not + groundtruth_is_group_of_list: A boolean numpy array of length M denoting + whether a ground truth box is a group-of box or not + """ + for class_index in range(self.num_class): + num_gt_instances = np.sum( + groundtruth_class_labels[ + ~groundtruth_is_difficult_list + & ~groundtruth_is_group_of_list + ] + == class_index + ) + self.num_gt_instances_per_class[class_index] += num_gt_instances + if np.any(groundtruth_class_labels == class_index): + self.num_gt_imgs_per_class[class_index] += 1 + + def evaluate(self): + """Compute evaluation result. + + Returns: + A named tuple with the following fields - + average_precision: float numpy array of average precision for + each class. + mean_ap: mean average precision of all classes, float scalar + precisions: List of precisions, each precision is a float numpy + array + recalls: List of recalls, each recall is a float numpy array + corloc: numpy float array + mean_corloc: Mean CorLoc score for each class, float scalar + """ + if (self.num_gt_instances_per_class == 0).any(): + logging.info( + "The following classes have no ground truth examples: %s", + np.squeeze(np.argwhere(self.num_gt_instances_per_class == 0)) + + self.label_id_offset, + ) + + if self.use_weighted_mean_ap: + all_scores = np.array([], dtype=float) + all_tp_fp_labels = np.array([], dtype=bool) + + for class_index in range(self.num_class): + if self.num_gt_instances_per_class[class_index] == 0: + continue + if not self.scores_per_class[class_index]: + scores = np.array([], dtype=float) + tp_fp_labels = np.array([], dtype=bool) + else: + scores = np.concatenate(self.scores_per_class[class_index]) + tp_fp_labels = np.concatenate( + self.tp_fp_labels_per_class[class_index] + ) + if self.use_weighted_mean_ap: + all_scores = np.append(all_scores, scores) + all_tp_fp_labels = np.append(all_tp_fp_labels, tp_fp_labels) + precision, recall = metrics.compute_precision_recall( + scores, + tp_fp_labels, + self.num_gt_instances_per_class[class_index], + ) + self.precisions_per_class.append(precision) + self.recalls_per_class.append(recall) + average_precision = metrics.compute_average_precision( + precision, recall + ) + self.average_precision_per_class[class_index] = average_precision + + self.corloc_per_class = metrics.compute_cor_loc( + self.num_gt_imgs_per_class, + self.num_images_correctly_detected_per_class, + ) + + if self.use_weighted_mean_ap: + num_gt_instances = np.sum(self.num_gt_instances_per_class) + precision, recall = metrics.compute_precision_recall( + all_scores, all_tp_fp_labels, num_gt_instances + ) + mean_ap = metrics.compute_average_precision(precision, recall) + else: + mean_ap = np.nanmean(self.average_precision_per_class) + mean_corloc = np.nanmean(self.corloc_per_class) + return ObjectDetectionEvalMetrics( + self.average_precision_per_class, + mean_ap, + self.precisions_per_class, + self.recalls_per_class, + self.corloc_per_class, + mean_corloc, + ) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/per_image_evaluation.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/per_image_evaluation.py" new file mode 100644 index 0000000..aa5684e --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/per_image_evaluation.py" @@ -0,0 +1,453 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Evaluate Object Detection result on a single image. + +Annotate each detected result as true positives or false positive according to +a predefined IOU ratio. Non Maximum Supression is used by default. Multi class +detection is supported by default. +Based on the settings, per image evaluation is either performed on boxes or +on object masks. +""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) +import numpy as np + +from . import ( + np_box_list, + np_box_list_ops, + np_box_mask_list, + np_box_mask_list_ops, +) + + +class PerImageEvaluation(object): + """Evaluate detection result of a single image.""" + + def __init__(self, num_groundtruth_classes, matching_iou_threshold=0.5): + """Initialized PerImageEvaluation by evaluation parameters. + + Args: + num_groundtruth_classes: Number of ground truth object classes + matching_iou_threshold: A ratio of area intersection to union, which is + the threshold to consider whether a detection is true positive or not + """ + self.matching_iou_threshold = matching_iou_threshold + self.num_groundtruth_classes = num_groundtruth_classes + + def compute_object_detection_metrics( + self, + detected_boxes, + detected_scores, + detected_class_labels, + groundtruth_boxes, + groundtruth_class_labels, + groundtruth_is_difficult_list, + groundtruth_is_group_of_list, + detected_masks=None, + groundtruth_masks=None, + ): + """Evaluates detections as being tp, fp or ignored from a single image. + + The evaluation is done in two stages: + 1. All detections are matched to non group-of boxes; true positives are + determined and detections matched to difficult boxes are ignored. + 2. Detections that are determined as false positives are matched against + group-of boxes and ignored if matched. + + Args: + detected_boxes: A float numpy array of shape [N, 4], representing N + regions of detected object regions. + Each row is of the format [y_min, x_min, y_max, x_max] + detected_scores: A float numpy array of shape [N, 1], representing + the confidence scores of the detected N object instances. + detected_class_labels: A integer numpy array of shape [N, 1], repreneting + the class labels of the detected N object instances. + groundtruth_boxes: A float numpy array of shape [M, 4], representing M + regions of object instances in ground truth + groundtruth_class_labels: An integer numpy array of shape [M, 1], + representing M class labels of object instances in ground truth + groundtruth_is_difficult_list: A boolean numpy array of length M denoting + whether a ground truth box is a difficult instance or not + groundtruth_is_group_of_list: A boolean numpy array of length M denoting + whether a ground truth box has group-of tag + detected_masks: (optional) A uint8 numpy array of shape + [N, height, width]. If not None, the metrics will be computed based + on masks. + groundtruth_masks: (optional) A uint8 numpy array of shape + [M, height, width]. + + Returns: + scores: A list of C float numpy arrays. Each numpy array is of + shape [K, 1], representing K scores detected with object class + label c + tp_fp_labels: A list of C boolean numpy arrays. Each numpy array + is of shape [K, 1], representing K True/False positive label of + object instances detected with class label c + """ + ( + detected_boxes, + detected_scores, + detected_class_labels, + detected_masks, + ) = self._remove_invalid_boxes( + detected_boxes, + detected_scores, + detected_class_labels, + detected_masks, + ) + scores, tp_fp_labels = self._compute_tp_fp( + detected_boxes=detected_boxes, + detected_scores=detected_scores, + detected_class_labels=detected_class_labels, + groundtruth_boxes=groundtruth_boxes, + groundtruth_class_labels=groundtruth_class_labels, + groundtruth_is_difficult_list=groundtruth_is_difficult_list, + groundtruth_is_group_of_list=groundtruth_is_group_of_list, + detected_masks=detected_masks, + groundtruth_masks=groundtruth_masks, + ) + + return scores, tp_fp_labels + + def _compute_tp_fp( + self, + detected_boxes, + detected_scores, + detected_class_labels, + groundtruth_boxes, + groundtruth_class_labels, + groundtruth_is_difficult_list, + groundtruth_is_group_of_list, + detected_masks=None, + groundtruth_masks=None, + ): + """Labels true/false positives of detections of an image across all classes. + + Args: + detected_boxes: A float numpy array of shape [N, 4], representing N + regions of detected object regions. + Each row is of the format [y_min, x_min, y_max, x_max] + detected_scores: A float numpy array of shape [N, 1], representing + the confidence scores of the detected N object instances. + detected_class_labels: A integer numpy array of shape [N, 1], repreneting + the class labels of the detected N object instances. + groundtruth_boxes: A float numpy array of shape [M, 4], representing M + regions of object instances in ground truth + groundtruth_class_labels: An integer numpy array of shape [M, 1], + representing M class labels of object instances in ground truth + groundtruth_is_difficult_list: A boolean numpy array of length M denoting + whether a ground truth box is a difficult instance or not + groundtruth_is_group_of_list: A boolean numpy array of length M denoting + whether a ground truth box has group-of tag + detected_masks: (optional) A np.uint8 numpy array of shape + [N, height, width]. If not None, the scores will be computed based + on masks. + groundtruth_masks: (optional) A np.uint8 numpy array of shape + [M, height, width]. + + Returns: + result_scores: A list of float numpy arrays. Each numpy array is of + shape [K, 1], representing K scores detected with object class + label c + result_tp_fp_labels: A list of boolean numpy array. Each numpy array is of + shape [K, 1], representing K True/False positive label of object + instances detected with class label c + + Raises: + ValueError: If detected masks is not None but groundtruth masks are None, + or the other way around. + """ + if detected_masks is not None and groundtruth_masks is None: + raise ValueError( + "Detected masks is available but groundtruth masks is not." + ) + if detected_masks is None and groundtruth_masks is not None: + raise ValueError( + "Groundtruth masks is available but detected masks is not." + ) + + result_scores = [] + result_tp_fp_labels = [] + for i in range(self.num_groundtruth_classes): + groundtruth_is_difficult_list_at_ith_class = ( + groundtruth_is_difficult_list[groundtruth_class_labels == i] + ) + groundtruth_is_group_of_list_at_ith_class = ( + groundtruth_is_group_of_list[groundtruth_class_labels == i] + ) + ( + gt_boxes_at_ith_class, + gt_masks_at_ith_class, + detected_boxes_at_ith_class, + detected_scores_at_ith_class, + detected_masks_at_ith_class, + ) = self._get_ith_class_arrays( + detected_boxes, + detected_scores, + detected_masks, + detected_class_labels, + groundtruth_boxes, + groundtruth_masks, + groundtruth_class_labels, + i, + ) + scores, tp_fp_labels = self._compute_tp_fp_for_single_class( + detected_boxes=detected_boxes_at_ith_class, + detected_scores=detected_scores_at_ith_class, + groundtruth_boxes=gt_boxes_at_ith_class, + groundtruth_is_difficult_list=groundtruth_is_difficult_list_at_ith_class, + groundtruth_is_group_of_list=groundtruth_is_group_of_list_at_ith_class, + detected_masks=detected_masks_at_ith_class, + groundtruth_masks=gt_masks_at_ith_class, + ) + result_scores.append(scores) + result_tp_fp_labels.append(tp_fp_labels) + return result_scores, result_tp_fp_labels + + def _get_overlaps_and_scores_box_mode( + self, + detected_boxes, + detected_scores, + groundtruth_boxes, + groundtruth_is_group_of_list, + ): + """Computes overlaps and scores between detected and groudntruth boxes. + + Args: + detected_boxes: A numpy array of shape [N, 4] representing detected box + coordinates + detected_scores: A 1-d numpy array of length N representing classification + score + groundtruth_boxes: A numpy array of shape [M, 4] representing ground truth + box coordinates + groundtruth_is_group_of_list: A boolean numpy array of length M denoting + whether a ground truth box has group-of tag. If a groundtruth box + is group-of box, every detection matching this box is ignored. + + Returns: + iou: A float numpy array of size [num_detected_boxes, num_gt_boxes]. If + gt_non_group_of_boxlist.num_boxes() == 0 it will be None. + ioa: A float numpy array of size [num_detected_boxes, num_gt_boxes]. If + gt_group_of_boxlist.num_boxes() == 0 it will be None. + scores: The score of the detected boxlist. + num_boxes: Number of non-maximum suppressed detected boxes. + """ + detected_boxlist = np_box_list.BoxList(detected_boxes) + detected_boxlist.add_field("scores", detected_scores) + gt_non_group_of_boxlist = np_box_list.BoxList( + groundtruth_boxes[~groundtruth_is_group_of_list] + ) + iou = np_box_list_ops.iou(detected_boxlist, gt_non_group_of_boxlist) + scores = detected_boxlist.get_field("scores") + num_boxes = detected_boxlist.num_boxes() + return iou, None, scores, num_boxes + + def _compute_tp_fp_for_single_class( + self, + detected_boxes, + detected_scores, + groundtruth_boxes, + groundtruth_is_difficult_list, + groundtruth_is_group_of_list, + detected_masks=None, + groundtruth_masks=None, + ): + """Labels boxes detected with the same class from the same image as tp/fp. + + Args: + detected_boxes: A numpy array of shape [N, 4] representing detected box + coordinates + detected_scores: A 1-d numpy array of length N representing classification + score + groundtruth_boxes: A numpy array of shape [M, 4] representing ground truth + box coordinates + groundtruth_is_difficult_list: A boolean numpy array of length M denoting + whether a ground truth box is a difficult instance or not. If a + groundtruth box is difficult, every detection matching this box + is ignored. + groundtruth_is_group_of_list: A boolean numpy array of length M denoting + whether a ground truth box has group-of tag. If a groundtruth box + is group-of box, every detection matching this box is ignored. + detected_masks: (optional) A uint8 numpy array of shape + [N, height, width]. If not None, the scores will be computed based + on masks. + groundtruth_masks: (optional) A uint8 numpy array of shape + [M, height, width]. + + Returns: + Two arrays of the same size, containing all boxes that were evaluated as + being true positives or false positives; if a box matched to a difficult + box or to a group-of box, it is ignored. + + scores: A numpy array representing the detection scores. + tp_fp_labels: a boolean numpy array indicating whether a detection is a + true positive. + """ + if detected_boxes.size == 0: + return np.array([], dtype=float), np.array([], dtype=bool) + + ( + iou, + _, + scores, + num_detected_boxes, + ) = self._get_overlaps_and_scores_box_mode( + detected_boxes=detected_boxes, + detected_scores=detected_scores, + groundtruth_boxes=groundtruth_boxes, + groundtruth_is_group_of_list=groundtruth_is_group_of_list, + ) + + if groundtruth_boxes.size == 0: + return scores, np.zeros(num_detected_boxes, dtype=bool) + + tp_fp_labels = np.zeros(num_detected_boxes, dtype=bool) + is_matched_to_difficult_box = np.zeros(num_detected_boxes, dtype=bool) + is_matched_to_group_of_box = np.zeros(num_detected_boxes, dtype=bool) + + # The evaluation is done in two stages: + # 1. All detections are matched to non group-of boxes; true positives are + # determined and detections matched to difficult boxes are ignored. + # 2. Detections that are determined as false positives are matched against + # group-of boxes and ignored if matched. + + # Tp-fp evaluation for non-group of boxes (if any). + if iou.shape[1] > 0: + groundtruth_nongroup_of_is_difficult_list = ( + groundtruth_is_difficult_list[~groundtruth_is_group_of_list] + ) + max_overlap_gt_ids = np.argmax(iou, axis=1) + is_gt_box_detected = np.zeros(iou.shape[1], dtype=bool) + for i in range(num_detected_boxes): + gt_id = max_overlap_gt_ids[i] + if iou[i, gt_id] >= self.matching_iou_threshold: + if not groundtruth_nongroup_of_is_difficult_list[gt_id]: + if not is_gt_box_detected[gt_id]: + tp_fp_labels[i] = True + is_gt_box_detected[gt_id] = True + else: + is_matched_to_difficult_box[i] = True + + return ( + scores[~is_matched_to_difficult_box & ~is_matched_to_group_of_box], + tp_fp_labels[ + ~is_matched_to_difficult_box & ~is_matched_to_group_of_box + ], + ) + + def _get_ith_class_arrays( + self, + detected_boxes, + detected_scores, + detected_masks, + detected_class_labels, + groundtruth_boxes, + groundtruth_masks, + groundtruth_class_labels, + class_index, + ): + """Returns numpy arrays belonging to class with index `class_index`. + + Args: + detected_boxes: A numpy array containing detected boxes. + detected_scores: A numpy array containing detected scores. + detected_masks: A numpy array containing detected masks. + detected_class_labels: A numpy array containing detected class labels. + groundtruth_boxes: A numpy array containing groundtruth boxes. + groundtruth_masks: A numpy array containing groundtruth masks. + groundtruth_class_labels: A numpy array containing groundtruth class + labels. + class_index: An integer index. + + Returns: + gt_boxes_at_ith_class: A numpy array containing groundtruth boxes labeled + as ith class. + gt_masks_at_ith_class: A numpy array containing groundtruth masks labeled + as ith class. + detected_boxes_at_ith_class: A numpy array containing detected boxes + corresponding to the ith class. + detected_scores_at_ith_class: A numpy array containing detected scores + corresponding to the ith class. + detected_masks_at_ith_class: A numpy array containing detected masks + corresponding to the ith class. + """ + selected_groundtruth = groundtruth_class_labels == class_index + gt_boxes_at_ith_class = groundtruth_boxes[selected_groundtruth] + if groundtruth_masks is not None: + gt_masks_at_ith_class = groundtruth_masks[selected_groundtruth] + else: + gt_masks_at_ith_class = None + selected_detections = detected_class_labels == class_index + detected_boxes_at_ith_class = detected_boxes[selected_detections] + detected_scores_at_ith_class = detected_scores[selected_detections] + if detected_masks is not None: + detected_masks_at_ith_class = detected_masks[selected_detections] + else: + detected_masks_at_ith_class = None + return ( + gt_boxes_at_ith_class, + gt_masks_at_ith_class, + detected_boxes_at_ith_class, + detected_scores_at_ith_class, + detected_masks_at_ith_class, + ) + + def _remove_invalid_boxes( + self, + detected_boxes, + detected_scores, + detected_class_labels, + detected_masks=None, + ): + """Removes entries with invalid boxes. + + A box is invalid if either its xmax is smaller than its xmin, or its ymax + is smaller than its ymin. + + Args: + detected_boxes: A float numpy array of size [num_boxes, 4] containing box + coordinates in [ymin, xmin, ymax, xmax] format. + detected_scores: A float numpy array of size [num_boxes]. + detected_class_labels: A int32 numpy array of size [num_boxes]. + detected_masks: A uint8 numpy array of size [num_boxes, height, width]. + + Returns: + valid_detected_boxes: A float numpy array of size [num_valid_boxes, 4] + containing box coordinates in [ymin, xmin, ymax, xmax] format. + valid_detected_scores: A float numpy array of size [num_valid_boxes]. + valid_detected_class_labels: A int32 numpy array of size + [num_valid_boxes]. + valid_detected_masks: A uint8 numpy array of size + [num_valid_boxes, height, width]. + """ + valid_indices = np.logical_and( + detected_boxes[:, 0] < detected_boxes[:, 2], + detected_boxes[:, 1] < detected_boxes[:, 3], + ) + detected_boxes = detected_boxes[valid_indices] + detected_scores = detected_scores[valid_indices] + detected_class_labels = detected_class_labels[valid_indices] + if detected_masks is not None: + detected_masks = detected_masks[valid_indices] + return [ + detected_boxes, + detected_scores, + detected_class_labels, + detected_masks, + ] diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/standard_fields.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/standard_fields.py" new file mode 100644 index 0000000..5b1a3df --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/ava_evaluation/standard_fields.py" @@ -0,0 +1,225 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Contains classes specifying naming conventions used for object detection. + + +Specifies: + InputDataFields: standard fields used by reader/preprocessor/batcher. + DetectionResultFields: standard fields returned by object detector. + BoxListFields: standard field used by BoxList + TfExampleFields: standard fields for tf-example data format (go/tf-example). +""" + + +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + + +class InputDataFields(object): + """Names for the input tensors. + + Holds the standard data field names to use for identifying input tensors. This + should be used by the decoder to identify keys for the returned tensor_dict + containing input tensors. And it should be used by the model to identify the + tensors it needs. + + Attributes: + image: image. + original_image: image in the original input size. + key: unique key corresponding to image. + source_id: source of the original image. + filename: original filename of the dataset (without common path). + groundtruth_image_classes: image-level class labels. + groundtruth_boxes: coordinates of the ground truth boxes in the image. + groundtruth_classes: box-level class labels. + groundtruth_label_types: box-level label types (e.g. explicit negative). + groundtruth_is_crowd: [DEPRECATED, use groundtruth_group_of instead] + is the groundtruth a single object or a crowd. + groundtruth_area: area of a groundtruth segment. + groundtruth_difficult: is a `difficult` object + groundtruth_group_of: is a `group_of` objects, e.g. multiple objects of the + same class, forming a connected group, where instances are heavily + occluding each other. + proposal_boxes: coordinates of object proposal boxes. + proposal_objectness: objectness score of each proposal. + groundtruth_instance_masks: ground truth instance masks. + groundtruth_instance_boundaries: ground truth instance boundaries. + groundtruth_instance_classes: instance mask-level class labels. + groundtruth_keypoints: ground truth keypoints. + groundtruth_keypoint_visibilities: ground truth keypoint visibilities. + groundtruth_label_scores: groundtruth label scores. + groundtruth_weights: groundtruth weight factor for bounding boxes. + num_groundtruth_boxes: number of groundtruth boxes. + true_image_shapes: true shapes of images in the resized images, as resized + images can be padded with zeros. + """ + + image = "image" + original_image = "original_image" + key = "key" + source_id = "source_id" + filename = "filename" + groundtruth_image_classes = "groundtruth_image_classes" + groundtruth_boxes = "groundtruth_boxes" + groundtruth_classes = "groundtruth_classes" + groundtruth_label_types = "groundtruth_label_types" + groundtruth_is_crowd = "groundtruth_is_crowd" + groundtruth_area = "groundtruth_area" + groundtruth_difficult = "groundtruth_difficult" + groundtruth_group_of = "groundtruth_group_of" + proposal_boxes = "proposal_boxes" + proposal_objectness = "proposal_objectness" + groundtruth_instance_masks = "groundtruth_instance_masks" + groundtruth_instance_boundaries = "groundtruth_instance_boundaries" + groundtruth_instance_classes = "groundtruth_instance_classes" + groundtruth_keypoints = "groundtruth_keypoints" + groundtruth_keypoint_visibilities = "groundtruth_keypoint_visibilities" + groundtruth_label_scores = "groundtruth_label_scores" + groundtruth_weights = "groundtruth_weights" + num_groundtruth_boxes = "num_groundtruth_boxes" + true_image_shape = "true_image_shape" + + +class DetectionResultFields(object): + """Naming conventions for storing the output of the detector. + + Attributes: + source_id: source of the original image. + key: unique key corresponding to image. + detection_boxes: coordinates of the detection boxes in the image. + detection_scores: detection scores for the detection boxes in the image. + detection_classes: detection-level class labels. + detection_masks: contains a segmentation mask for each detection box. + detection_boundaries: contains an object boundary for each detection box. + detection_keypoints: contains detection keypoints for each detection box. + num_detections: number of detections in the batch. + """ + + source_id = "source_id" + key = "key" + detection_boxes = "detection_boxes" + detection_scores = "detection_scores" + detection_classes = "detection_classes" + detection_masks = "detection_masks" + detection_boundaries = "detection_boundaries" + detection_keypoints = "detection_keypoints" + num_detections = "num_detections" + + +class BoxListFields(object): + """Naming conventions for BoxLists. + + Attributes: + boxes: bounding box coordinates. + classes: classes per bounding box. + scores: scores per bounding box. + weights: sample weights per bounding box. + objectness: objectness score per bounding box. + masks: masks per bounding box. + boundaries: boundaries per bounding box. + keypoints: keypoints per bounding box. + keypoint_heatmaps: keypoint heatmaps per bounding box. + """ + + boxes = "boxes" + classes = "classes" + scores = "scores" + weights = "weights" + objectness = "objectness" + masks = "masks" + boundaries = "boundaries" + keypoints = "keypoints" + keypoint_heatmaps = "keypoint_heatmaps" + + +class TfExampleFields(object): + """TF-example proto feature names for object detection. + + Holds the standard feature names to load from an Example proto for object + detection. + + Attributes: + image_encoded: JPEG encoded string + image_format: image format, e.g. "JPEG" + filename: filename + channels: number of channels of image + colorspace: colorspace, e.g. "RGB" + height: height of image in pixels, e.g. 462 + width: width of image in pixels, e.g. 581 + source_id: original source of the image + object_class_text: labels in text format, e.g. ["person", "cat"] + object_class_label: labels in numbers, e.g. [16, 8] + object_bbox_xmin: xmin coordinates of groundtruth box, e.g. 10, 30 + object_bbox_xmax: xmax coordinates of groundtruth box, e.g. 50, 40 + object_bbox_ymin: ymin coordinates of groundtruth box, e.g. 40, 50 + object_bbox_ymax: ymax coordinates of groundtruth box, e.g. 80, 70 + object_view: viewpoint of object, e.g. ["frontal", "left"] + object_truncated: is object truncated, e.g. [true, false] + object_occluded: is object occluded, e.g. [true, false] + object_difficult: is object difficult, e.g. [true, false] + object_group_of: is object a single object or a group of objects + object_depiction: is object a depiction + object_is_crowd: [DEPRECATED, use object_group_of instead] + is the object a single object or a crowd + object_segment_area: the area of the segment. + object_weight: a weight factor for the object's bounding box. + instance_masks: instance segmentation masks. + instance_boundaries: instance boundaries. + instance_classes: Classes for each instance segmentation mask. + detection_class_label: class label in numbers. + detection_bbox_ymin: ymin coordinates of a detection box. + detection_bbox_xmin: xmin coordinates of a detection box. + detection_bbox_ymax: ymax coordinates of a detection box. + detection_bbox_xmax: xmax coordinates of a detection box. + detection_score: detection score for the class label and box. + """ + + image_encoded = "image/encoded" + image_format = "image/format" # format is reserved keyword + filename = "image/filename" + channels = "image/channels" + colorspace = "image/colorspace" + height = "image/height" + width = "image/width" + source_id = "image/source_id" + object_class_text = "image/object/class/text" + object_class_label = "image/object/class/label" + object_bbox_ymin = "image/object/bbox/ymin" + object_bbox_xmin = "image/object/bbox/xmin" + object_bbox_ymax = "image/object/bbox/ymax" + object_bbox_xmax = "image/object/bbox/xmax" + object_view = "image/object/view" + object_truncated = "image/object/truncated" + object_occluded = "image/object/occluded" + object_difficult = "image/object/difficult" + object_group_of = "image/object/group_of" + object_depiction = "image/object/depiction" + object_is_crowd = "image/object/is_crowd" + object_segment_area = "image/object/segment/area" + object_weight = "image/object/weight" + instance_masks = "image/segmentation/object" + instance_boundaries = "image/boundaries/object" + instance_classes = "image/segmentation/object/class" + detection_class_label = "image/detection/label" + detection_bbox_ymin = "image/detection/bbox/ymin" + detection_bbox_xmin = "image/detection/bbox/xmin" + detection_bbox_ymax = "image/detection/bbox/ymax" + detection_bbox_xmax = "image/detection/bbox/xmax" + detection_score = "image/detection/score" diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/benchmark.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/benchmark.py" new file mode 100644 index 0000000..33e5fe9 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/benchmark.py" @@ -0,0 +1,103 @@ +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +Functions for benchmarks. +""" + +import numpy as np +import pprint +import torch +import tqdm +from fvcore.common.timer import Timer + +import slowfast.utils.logging as logging +import slowfast.utils.misc as misc +from slowfast.datasets import loader +from slowfast.utils.env import setup_environment + +logger = logging.get_logger(__name__) + + +def benchmark_data_loading(cfg): + """ + Benchmark the speed of data loading in PySlowFast. + Args: + + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + # Set up environment. + setup_environment() + # Set random seed from configs. + np.random.seed(cfg.RNG_SEED) + torch.manual_seed(cfg.RNG_SEED) + + # Setup logging format. + logging.setup_logging(cfg.OUTPUT_DIR) + + # Print config. + logger.info("Benchmark data loading with config:") + logger.info(pprint.pformat(cfg)) + + timer = Timer() + dataloader = loader.construct_loader(cfg, "train") + logger.info( + "Initialize loader using {:.2f} seconds.".format(timer.seconds()) + ) + # Total batch size across different machines. + batch_size = cfg.TRAIN.BATCH_SIZE * cfg.NUM_SHARDS + log_period = cfg.BENCHMARK.LOG_PERIOD + epoch_times = [] + # Test for a few epochs. + for cur_epoch in range(cfg.BENCHMARK.NUM_EPOCHS): + timer = Timer() + timer_epoch = Timer() + iter_times = [] + if cfg.BENCHMARK.SHUFFLE: + loader.shuffle_dataset(dataloader, cur_epoch) + for cur_iter, _ in enumerate(tqdm.tqdm(dataloader)): + if cur_iter > 0 and cur_iter % log_period == 0: + iter_times.append(timer.seconds()) + ram_usage, ram_total = misc.cpu_mem_usage() + logger.info( + "Epoch {}: {} iters ({} videos) in {:.2f} seconds. " + "RAM Usage: {:.2f}/{:.2f} GB.".format( + cur_epoch, + log_period, + log_period * batch_size, + iter_times[-1], + ram_usage, + ram_total, + ) + ) + timer.reset() + epoch_times.append(timer_epoch.seconds()) + ram_usage, ram_total = misc.cpu_mem_usage() + logger.info( + "Epoch {}: in total {} iters ({} videos) in {:.2f} seconds. " + "RAM Usage: {:.2f}/{:.2f} GB.".format( + cur_epoch, + len(dataloader), + len(dataloader) * batch_size, + epoch_times[-1], + ram_usage, + ram_total, + ) + ) + logger.info( + "Epoch {}: on average every {} iters ({} videos) take {:.2f}/{:.2f} " + "(avg/std) seconds.".format( + cur_epoch, + log_period, + log_period * batch_size, + np.mean(iter_times), + np.std(iter_times), + ) + ) + logger.info( + "On average every epoch ({} videos) takes {:.2f}/{:.2f} " + "(avg/std) seconds.".format( + len(dataloader) * batch_size, + np.mean(epoch_times), + np.std(epoch_times), + ) + ) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/bn_helper.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/bn_helper.py" new file mode 100644 index 0000000..b18d8c7 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/bn_helper.py" @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""bn helper.""" + +import itertools +import torch + + +@torch.no_grad() +def compute_and_update_bn_stats(model, data_loader, num_batches=200): + """ + Compute and update the batch norm stats to make it more precise. During + training both bn stats and the weight are changing after every iteration, + so the bn can not precisely reflect the latest stats of the current model. + Here the bn stats is recomputed without change of weights, to make the + running mean and running var more precise. + Args: + model (model): the model using to compute and update the bn stats. + data_loader (dataloader): dataloader using to provide inputs. + num_batches (int): running iterations using to compute the stats. + """ + + # Prepares all the bn layers. + bn_layers = [ + m + for m in model.modules() + if any( + ( + isinstance(m, bn_type) + for bn_type in ( + torch.nn.BatchNorm1d, + torch.nn.BatchNorm2d, + torch.nn.BatchNorm3d, + ) + ) + ) + ] + + # In order to make the running stats only reflect the current batch, the + # momentum is disabled. + # bn.running_mean = (1 - momentum) * bn.running_mean + momentum * batch_mean + # Setting the momentum to 1.0 to compute the stats without momentum. + momentum_actual = [bn.momentum for bn in bn_layers] + for bn in bn_layers: + bn.momentum = 1.0 + + # Calculates the running iterations for precise stats computation. + running_mean = [torch.zeros_like(bn.running_mean) for bn in bn_layers] + running_square_mean = [torch.zeros_like(bn.running_var) for bn in bn_layers] + + for ind, (inputs, _, _) in enumerate( + itertools.islice(data_loader, num_batches) + ): + # Forwards the model to update the bn stats. + if isinstance(inputs, (list,)): + for i in range(len(inputs)): + inputs[i] = inputs[i].float().cuda(non_blocking=True) + else: + inputs = inputs.cuda(non_blocking=True) + model(inputs) + + for i, bn in enumerate(bn_layers): + # Accumulates the bn stats. + running_mean[i] += (bn.running_mean - running_mean[i]) / (ind + 1) + # $E(x^2) = Var(x) + E(x)^2$. + cur_square_mean = bn.running_var + bn.running_mean ** 2 + running_square_mean[i] += ( + cur_square_mean - running_square_mean[i] + ) / (ind + 1) + + for i, bn in enumerate(bn_layers): + bn.running_mean = running_mean[i] + # Var(x) = $E(x^2) - E(x)^2$. + bn.running_var = running_square_mean[i] - bn.running_mean ** 2 + # Sets the precise bn stats. + bn.momentum = momentum_actual[i] diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/c2_model_loading.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/c2_model_loading.py" new file mode 100644 index 0000000..4ca42b8 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/c2_model_loading.py" @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Caffe2 to PyTorch checkpoint name converting utility.""" + +import re + + +def get_name_convert_func(): + """ + Get the function to convert Caffe2 layer names to PyTorch layer names. + Returns: + (func): function to convert parameter name from Caffe2 format to PyTorch + format. + """ + pairs = [ + # ------------------------------------------------------------ + # 'nonlocal_conv3_1_theta_w' -> 's3.pathway0_nonlocal3.conv_g.weight' + [ + r"^nonlocal_conv([0-9]+)_([0-9]+)_(.*)", + r"s\1.pathway0_nonlocal\2_\3", + ], + # 'theta' -> 'conv_theta' + [r"^(.*)_nonlocal([0-9]+)_(theta)(.*)", r"\1_nonlocal\2.conv_\3\4"], + # 'g' -> 'conv_g' + [r"^(.*)_nonlocal([0-9]+)_(g)(.*)", r"\1_nonlocal\2.conv_\3\4"], + # 'phi' -> 'conv_phi' + [r"^(.*)_nonlocal([0-9]+)_(phi)(.*)", r"\1_nonlocal\2.conv_\3\4"], + # 'out' -> 'conv_out' + [r"^(.*)_nonlocal([0-9]+)_(out)(.*)", r"\1_nonlocal\2.conv_\3\4"], + # 'nonlocal_conv4_5_bn_s' -> 's4.pathway0_nonlocal3.bn.weight' + [r"^(.*)_nonlocal([0-9]+)_(bn)_(.*)", r"\1_nonlocal\2.\3.\4"], + # ------------------------------------------------------------ + # 't_pool1_subsample_bn' -> 's1_fuse.conv_f2s.bn.running_mean' + [r"^t_pool1_subsample_bn_(.*)", r"s1_fuse.bn.\1"], + # 't_pool1_subsample' -> 's1_fuse.conv_f2s' + [r"^t_pool1_subsample_(.*)", r"s1_fuse.conv_f2s.\1"], + # 't_res4_5_branch2c_bn_subsample_bn_rm' -> 's4_fuse.conv_f2s.bias' + [ + r"^t_res([0-9]+)_([0-9]+)_branch2c_bn_subsample_bn_(.*)", + r"s\1_fuse.bn.\3", + ], + # 't_pool1_subsample' -> 's1_fuse.conv_f2s' + [ + r"^t_res([0-9]+)_([0-9]+)_branch2c_bn_subsample_(.*)", + r"s\1_fuse.conv_f2s.\3", + ], + # ------------------------------------------------------------ + # 'res4_4_branch_2c_bn_b' -> 's4.pathway0_res4.branch2.c_bn_b' + [ + r"^res([0-9]+)_([0-9]+)_branch([0-9]+)([a-z])_(.*)", + r"s\1.pathway0_res\2.branch\3.\4_\5", + ], + # 'res_conv1_bn_' -> 's1.pathway0_stem.bn.' + [r"^res_conv1_bn_(.*)", r"s1.pathway0_stem.bn.\1"], + # 'conv1_xy_w_momentum' -> 's1.pathway0_stem.conv_xy.' + [r"^conv1_xy(.*)", r"s1.pathway0_stem.conv_xy\1"], + # 'conv1_w_momentum' -> 's1.pathway0_stem.conv.' + [r"^conv1_(.*)", r"s1.pathway0_stem.conv.\1"], + # 'res4_0_branch1_w' -> 'S4.pathway0_res0.branch1.weight' + [ + r"^res([0-9]+)_([0-9]+)_branch([0-9]+)_(.*)", + r"s\1.pathway0_res\2.branch\3_\4", + ], + # 'res_conv1_' -> 's1.pathway0_stem.conv.' + [r"^res_conv1_(.*)", r"s1.pathway0_stem.conv.\1"], + # ------------------------------------------------------------ + # 'res4_4_branch_2c_bn_b' -> 's4.pathway0_res4.branch2.c_bn_b' + [ + r"^t_res([0-9]+)_([0-9]+)_branch([0-9]+)([a-z])_(.*)", + r"s\1.pathway1_res\2.branch\3.\4_\5", + ], + # 'res_conv1_bn_' -> 's1.pathway0_stem.bn.' + [r"^t_res_conv1_bn_(.*)", r"s1.pathway1_stem.bn.\1"], + # 'conv1_w_momentum' -> 's1.pathway0_stem.conv.' + [r"^t_conv1_(.*)", r"s1.pathway1_stem.conv.\1"], + # 'res4_0_branch1_w' -> 'S4.pathway0_res0.branch1.weight' + [ + r"^t_res([0-9]+)_([0-9]+)_branch([0-9]+)_(.*)", + r"s\1.pathway1_res\2.branch\3_\4", + ], + # 'res_conv1_' -> 's1.pathway0_stem.conv.' + [r"^t_res_conv1_(.*)", r"s1.pathway1_stem.conv.\1"], + # ------------------------------------------------------------ + # pred_ -> head.projection. + [r"pred_(.*)", r"head.projection.\1"], + # '.b_bn_fc' -> '.se.fc' + [r"(.*)b_bn_fc(.*)", r"\1se.fc\2"], + # conv_5 -> head.conv_5. + [r"conv_5(.*)", r"head.conv_5\1"], + # conv_5 -> head.conv_5. + [r"lin_5(.*)", r"head.lin_5\1"], + # '.bn_b' -> '.weight' + [r"(.*)bn.b\Z", r"\1bn.bias"], + # '.bn_s' -> '.weight' + [r"(.*)bn.s\Z", r"\1bn.weight"], + # '_bn_rm' -> '.running_mean' + [r"(.*)bn.rm\Z", r"\1bn.running_mean"], + # '_bn_riv' -> '.running_var' + [r"(.*)bn.riv\Z", r"\1bn.running_var"], + # '_b' -> '.bias' + [r"(.*)[\._]b\Z", r"\1.bias"], + # '_w' -> '.weight' + [r"(.*)[\._]w\Z", r"\1.weight"], + ] + + def convert_caffe2_name_to_pytorch(caffe2_layer_name): + """ + Convert the caffe2_layer_name to pytorch format by apply the list of + regular expressions. + Args: + caffe2_layer_name (str): caffe2 layer name. + Returns: + (str): pytorch layer name. + """ + for source, dest in pairs: + caffe2_layer_name = re.sub(source, dest, caffe2_layer_name) + return caffe2_layer_name + + return convert_caffe2_name_to_pytorch diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/checkpoint.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/checkpoint.py" new file mode 100644 index 0000000..f3a7a9b --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/checkpoint.py" @@ -0,0 +1,514 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Functions that handle saving and loading of checkpoints.""" + +import copy +import numpy as np +import os +import pickle +from collections import OrderedDict +import torch +from iopath.common.file_io import g_pathmgr + +import slowfast.utils.distributed as du +import slowfast.utils.logging as logging +from slowfast.utils.c2_model_loading import get_name_convert_func + +logger = logging.get_logger(__name__) + + +def make_checkpoint_dir(path_to_job): + """ + Creates the checkpoint directory (if not present already). + Args: + path_to_job (string): the path to the folder of the current job. + """ + checkpoint_dir = os.path.join(path_to_job, "checkpoints") + # Create the checkpoint dir from the master process + if du.is_master_proc() and not g_pathmgr.exists(checkpoint_dir): + try: + g_pathmgr.mkdirs(checkpoint_dir) + except Exception: + pass + return checkpoint_dir + + +def get_checkpoint_dir(path_to_job): + """ + Get path for storing checkpoints. + Args: + path_to_job (string): the path to the folder of the current job. + """ + return os.path.join(path_to_job, "checkpoints") + + +def get_path_to_checkpoint(path_to_job, epoch): + """ + Get the full path to a checkpoint file. + Args: + path_to_job (string): the path to the folder of the current job. + epoch (int): the number of epoch for the checkpoint. + """ + name = "checkpoint_epoch_{:05d}.pyth".format(epoch) + return os.path.join(get_checkpoint_dir(path_to_job), name) + + +def get_last_checkpoint(path_to_job): + """ + Get the last checkpoint from the checkpointing folder. + Args: + path_to_job (string): the path to the folder of the current job. + """ + + d = get_checkpoint_dir(path_to_job) + names = g_pathmgr.ls(d) if g_pathmgr.exists(d) else [] + names = [f for f in names if "checkpoint" in f] + assert len(names), "No checkpoints found in '{}'.".format(d) + # Sort the checkpoints by epoch. + name = sorted(names)[-1] + return os.path.join(d, name) + + +def has_checkpoint(path_to_job): + """ + Determines if the given directory contains a checkpoint. + Args: + path_to_job (string): the path to the folder of the current job. + """ + d = get_checkpoint_dir(path_to_job) + files = g_pathmgr.ls(d) if g_pathmgr.exists(d) else [] + return any("checkpoint" in f for f in files) + + +def is_checkpoint_epoch(cfg, cur_epoch, multigrid_schedule=None): + """ + Determine if a checkpoint should be saved on current epoch. + Args: + cfg (CfgNode): configs to save. + cur_epoch (int): current number of epoch of the model. + multigrid_schedule (List): schedule for multigrid training. + """ + if cur_epoch + 1 == cfg.SOLVER.MAX_EPOCH: + return True + if multigrid_schedule is not None: + prev_epoch = 0 + for s in multigrid_schedule: + if cur_epoch < s[-1]: + period = max( + (s[-1] - prev_epoch) // cfg.MULTIGRID.EVAL_FREQ + 1, 1 + ) + return (s[-1] - 1 - cur_epoch) % period == 0 + prev_epoch = s[-1] + + return (cur_epoch + 1) % cfg.TRAIN.CHECKPOINT_PERIOD == 0 + + +def save_checkpoint(path_to_job, model, optimizer, epoch, cfg): + """ + Save a checkpoint. + Args: + model (model): model to save the weight to the checkpoint. + optimizer (optim): optimizer to save the historical state. + epoch (int): current number of epoch of the model. + cfg (CfgNode): configs to save. + """ + # Save checkpoints only from the master process. + if not du.is_master_proc(cfg.NUM_GPUS * cfg.NUM_SHARDS): + return + # Ensure that the checkpoint dir exists. + g_pathmgr.mkdirs(get_checkpoint_dir(path_to_job)) + # Omit the DDP wrapper in the multi-gpu setting. + sd = model.module.state_dict() if cfg.NUM_GPUS > 1 else model.state_dict() + normalized_sd = sub_to_normal_bn(sd) + + # Record the state. + checkpoint = { + "epoch": epoch, + "model_state": normalized_sd, + "optimizer_state": optimizer.state_dict(), + "cfg": cfg.dump(), + } + # Write the checkpoint. + path_to_checkpoint = get_path_to_checkpoint(path_to_job, epoch + 1) + with g_pathmgr.open(path_to_checkpoint, "wb") as f: + torch.save(checkpoint, f) + return path_to_checkpoint + + +def inflate_weight(state_dict_2d, state_dict_3d): + """ + Inflate 2D model weights in state_dict_2d to the 3D model weights in + state_dict_3d. The details can be found in: + Joao Carreira, and Andrew Zisserman. + "Quo vadis, action recognition? a new model and the kinetics dataset." + Args: + state_dict_2d (OrderedDict): a dict of parameters from a 2D model. + state_dict_3d (OrderedDict): a dict of parameters from a 3D model. + Returns: + state_dict_inflated (OrderedDict): a dict of inflated parameters. + """ + state_dict_inflated = OrderedDict() + for k, v2d in state_dict_2d.items(): + assert k in state_dict_3d.keys() + v3d = state_dict_3d[k] + # Inflate the weight of 2D conv to 3D conv. + if len(v2d.shape) == 4 and len(v3d.shape) == 5: + logger.info( + "Inflate {}: {} -> {}: {}".format(k, v2d.shape, k, v3d.shape) + ) + # Dimension need to be match. + assert v2d.shape[-2:] == v3d.shape[-2:] + assert v2d.shape[:2] == v3d.shape[:2] + v3d = ( + v2d.unsqueeze(2).repeat(1, 1, v3d.shape[2], 1, 1) / v3d.shape[2] + ) + elif v2d.shape == v3d.shape: + v3d = v2d + else: + logger.info( + "Unexpected {}: {} -|> {}: {}".format( + k, v2d.shape, k, v3d.shape + ) + ) + state_dict_inflated[k] = v3d.clone() + return state_dict_inflated + + +def load_checkpoint( + path_to_checkpoint, + model, + data_parallel=True, + optimizer=None, + inflation=False, + convert_from_caffe2=False, + epoch_reset=False, + clear_name_pattern=(), +): + """ + Load the checkpoint from the given file. If inflation is True, inflate the + 2D Conv weights from the checkpoint to 3D Conv. + Args: + path_to_checkpoint (string): path to the checkpoint to load. + model (model): model to load the weights from the checkpoint. + data_parallel (bool): if true, model is wrapped by + torch.nn.parallel.DistributedDataParallel. + optimizer (optim): optimizer to load the historical state. + inflation (bool): if True, inflate the weights from the checkpoint. + convert_from_caffe2 (bool): if True, load the model from caffe2 and + convert it to pytorch. + epoch_reset (bool): if True, reset #train iterations from the checkpoint. + clear_name_pattern (string): if given, this (sub)string will be cleared + from a layer name if it can be matched. + Returns: + (int): the number of training epoch of the checkpoint. + """ + assert g_pathmgr.exists( + path_to_checkpoint + ), "Checkpoint '{}' not found".format(path_to_checkpoint) + logger.info("Loading network weights from {}.".format(path_to_checkpoint)) + + # Account for the DDP wrapper in the multi-gpu setting. + ms = model.module if data_parallel else model + if convert_from_caffe2: + with g_pathmgr.open(path_to_checkpoint, "rb") as f: + caffe2_checkpoint = pickle.load(f, encoding="latin1") + state_dict = OrderedDict() + name_convert_func = get_name_convert_func() + for key in caffe2_checkpoint["blobs"].keys(): + converted_key = name_convert_func(key) + converted_key = c2_normal_to_sub_bn(converted_key, ms.state_dict()) + if converted_key in ms.state_dict(): + c2_blob_shape = caffe2_checkpoint["blobs"][key].shape + model_blob_shape = ms.state_dict()[converted_key].shape + + # expand shape dims if they differ (eg for converting linear to conv params) + if len(c2_blob_shape) < len(model_blob_shape): + c2_blob_shape += (1,) * ( + len(model_blob_shape) - len(c2_blob_shape) + ) + caffe2_checkpoint["blobs"][key] = np.reshape( + caffe2_checkpoint["blobs"][key], c2_blob_shape + ) + # Load BN stats to Sub-BN. + if ( + len(model_blob_shape) == 1 + and len(c2_blob_shape) == 1 + and model_blob_shape[0] > c2_blob_shape[0] + and model_blob_shape[0] % c2_blob_shape[0] == 0 + ): + caffe2_checkpoint["blobs"][key] = np.concatenate( + [caffe2_checkpoint["blobs"][key]] + * (model_blob_shape[0] // c2_blob_shape[0]) + ) + c2_blob_shape = caffe2_checkpoint["blobs"][key].shape + + if c2_blob_shape == tuple(model_blob_shape): + state_dict[converted_key] = torch.tensor( + caffe2_checkpoint["blobs"][key] + ).clone() + logger.info( + "{}: {} => {}: {}".format( + key, + c2_blob_shape, + converted_key, + tuple(model_blob_shape), + ) + ) + else: + logger.warn( + "!! {}: {} does not match {}: {}".format( + key, + c2_blob_shape, + converted_key, + tuple(model_blob_shape), + ) + ) + else: + if not any( + prefix in key for prefix in ["momentum", "lr", "model_iter"] + ): + logger.warn( + "!! {}: can not be converted, got {}".format( + key, converted_key + ) + ) + diff = set(ms.state_dict()) - set(state_dict) + diff = {d for d in diff if "num_batches_tracked" not in d} + if len(diff) > 0: + logger.warn("Not loaded {}".format(diff)) + ms.load_state_dict(state_dict, strict=False) + epoch = -1 + else: + # Load the checkpoint on CPU to avoid GPU mem spike. + with g_pathmgr.open(path_to_checkpoint, "rb") as f: + checkpoint = torch.load(f, map_location="cpu") + model_state_dict_3d = ( + model.module.state_dict() if data_parallel else model.state_dict() + ) + checkpoint["model_state"] = normal_to_sub_bn( + checkpoint["model_state"], model_state_dict_3d + ) + if inflation: + # Try to inflate the model. + inflated_model_dict = inflate_weight( + checkpoint["model_state"], model_state_dict_3d + ) + ms.load_state_dict(inflated_model_dict, strict=False) + else: + if clear_name_pattern: + for item in clear_name_pattern: + model_state_dict_new = OrderedDict() + for k in checkpoint["model_state"]: + if item in k: + k_re = k.replace(item, "") + model_state_dict_new[k_re] = checkpoint[ + "model_state" + ][k] + logger.info("renaming: {} -> {}".format(k, k_re)) + else: + model_state_dict_new[k] = checkpoint["model_state"][ + k + ] + checkpoint["model_state"] = model_state_dict_new + + pre_train_dict = checkpoint["model_state"] + model_dict = ms.state_dict() + # Match pre-trained weights that have same shape as current model. + pre_train_dict_match = { + k: v + for k, v in pre_train_dict.items() + if k in model_dict and v.size() == model_dict[k].size() + } + # Weights that do not have match from the pre-trained model. + not_load_layers = [ + k + for k in model_dict.keys() + if k not in pre_train_dict_match.keys() + ] + # Log weights that are not loaded with the pre-trained weights. + if not_load_layers: + for k in not_load_layers: + logger.info("Network weights {} not loaded.".format(k)) + # Load pre-trained weights. + ms.load_state_dict(pre_train_dict_match, strict=False) + epoch = -1 + + # Load the optimizer state (commonly not done when fine-tuning) + if "epoch" in checkpoint.keys() and not epoch_reset: + epoch = checkpoint["epoch"] + if optimizer: + optimizer.load_state_dict(checkpoint["optimizer_state"]) + else: + epoch = -1 + return epoch + + +def sub_to_normal_bn(sd): + """ + Convert the Sub-BN paprameters to normal BN parameters in a state dict. + There are two copies of BN layers in a Sub-BN implementation: `bn.bn` and + `bn.split_bn`. `bn.split_bn` is used during training and + "compute_precise_bn". Before saving or evaluation, its stats are copied to + `bn.bn`. We rename `bn.bn` to `bn` and store it to be consistent with normal + BN layers. + Args: + sd (OrderedDict): a dict of parameters whitch might contain Sub-BN + parameters. + Returns: + new_sd (OrderedDict): a dict with Sub-BN parameters reshaped to + normal parameters. + """ + new_sd = copy.deepcopy(sd) + modifications = [ + ("bn.bn.running_mean", "bn.running_mean"), + ("bn.bn.running_var", "bn.running_var"), + ("bn.split_bn.num_batches_tracked", "bn.num_batches_tracked"), + ] + to_remove = ["bn.bn.", ".split_bn."] + for key in sd: + for before, after in modifications: + if key.endswith(before): + new_key = key.split(before)[0] + after + new_sd[new_key] = new_sd.pop(key) + + for rm in to_remove: + if rm in key and key in new_sd: + del new_sd[key] + + for key in new_sd: + if key.endswith("bn.weight") or key.endswith("bn.bias"): + if len(new_sd[key].size()) == 4: + assert all(d == 1 for d in new_sd[key].size()[1:]) + new_sd[key] = new_sd[key][:, 0, 0, 0] + + return new_sd + + +def c2_normal_to_sub_bn(key, model_keys): + """ + Convert BN parameters to Sub-BN parameters if model contains Sub-BNs. + Args: + key (OrderedDict): source dict of parameters. + mdoel_key (OrderedDict): target dict of parameters. + Returns: + new_sd (OrderedDict): converted dict of parameters. + """ + if "bn.running_" in key: + if key in model_keys: + return key + + new_key = key.replace("bn.running_", "bn.split_bn.running_") + if new_key in model_keys: + return new_key + else: + return key + + +def normal_to_sub_bn(checkpoint_sd, model_sd): + """ + Convert BN parameters to Sub-BN parameters if model contains Sub-BNs. + Args: + checkpoint_sd (OrderedDict): source dict of parameters. + model_sd (OrderedDict): target dict of parameters. + Returns: + new_sd (OrderedDict): converted dict of parameters. + """ + for key in model_sd: + if key not in checkpoint_sd: + if "bn.split_bn." in key: + load_key = key.replace("bn.split_bn.", "bn.") + bn_key = key.replace("bn.split_bn.", "bn.bn.") + checkpoint_sd[key] = checkpoint_sd.pop(load_key) + checkpoint_sd[bn_key] = checkpoint_sd[key] + + for key in model_sd: + if key in checkpoint_sd: + model_blob_shape = model_sd[key].shape + c2_blob_shape = checkpoint_sd[key].shape + + if ( + len(model_blob_shape) == 1 + and len(c2_blob_shape) == 1 + and model_blob_shape[0] > c2_blob_shape[0] + and model_blob_shape[0] % c2_blob_shape[0] == 0 + ): + before_shape = checkpoint_sd[key].shape + checkpoint_sd[key] = torch.cat( + [checkpoint_sd[key]] + * (model_blob_shape[0] // c2_blob_shape[0]) + ) + logger.info( + "{} {} -> {}".format( + key, before_shape, checkpoint_sd[key].shape + ) + ) + return checkpoint_sd + + +def load_test_checkpoint(cfg, model): + """ + Loading checkpoint logic for testing. + """ + # Load a checkpoint to test if applicable. + if cfg.TEST.CHECKPOINT_FILE_PATH != "": + # If no checkpoint found in MODEL_VIS.CHECKPOINT_FILE_PATH or in the current + # checkpoint folder, try to load checkpoint from + # TEST.CHECKPOINT_FILE_PATH and test it. + load_checkpoint( + cfg.TEST.CHECKPOINT_FILE_PATH, + model, + cfg.NUM_GPUS > 1, + None, + inflation=False, + convert_from_caffe2=cfg.TEST.CHECKPOINT_TYPE == "caffe2", + ) + elif has_checkpoint(cfg.OUTPUT_DIR): + last_checkpoint = get_last_checkpoint(cfg.OUTPUT_DIR) + load_checkpoint(last_checkpoint, model, cfg.NUM_GPUS > 1) + elif cfg.TRAIN.CHECKPOINT_FILE_PATH != "": + # If no checkpoint found in TEST.CHECKPOINT_FILE_PATH or in the current + # checkpoint folder, try to load checkpoint from + # TRAIN.CHECKPOINT_FILE_PATH and test it. + load_checkpoint( + cfg.TRAIN.CHECKPOINT_FILE_PATH, + model, + cfg.NUM_GPUS > 1, + None, + inflation=False, + convert_from_caffe2=cfg.TRAIN.CHECKPOINT_TYPE == "caffe2", + ) + else: + logger.info( + "Unknown way of loading checkpoint. Using with random initialization, only for debugging." + ) + + +def load_train_checkpoint(cfg, model, optimizer): + """ + Loading checkpoint logic for training. + """ + if cfg.TRAIN.AUTO_RESUME and has_checkpoint(cfg.OUTPUT_DIR): + last_checkpoint = get_last_checkpoint(cfg.OUTPUT_DIR) + logger.info("Load from last checkpoint, {}.".format(last_checkpoint)) + checkpoint_epoch = load_checkpoint( + last_checkpoint, model, cfg.NUM_GPUS > 1, optimizer + ) + start_epoch = checkpoint_epoch + 1 + elif cfg.TRAIN.CHECKPOINT_FILE_PATH != "": + logger.info("Load from given checkpoint file.") + checkpoint_epoch = load_checkpoint( + cfg.TRAIN.CHECKPOINT_FILE_PATH, + model, + cfg.NUM_GPUS > 1, + optimizer, + inflation=cfg.TRAIN.CHECKPOINT_INFLATE, + convert_from_caffe2=cfg.TRAIN.CHECKPOINT_TYPE == "caffe2", + epoch_reset=cfg.TRAIN.CHECKPOINT_EPOCH_RESET, + clear_name_pattern=cfg.TRAIN.CHECKPOINT_CLEAR_NAME_PATTERN, + ) + start_epoch = checkpoint_epoch + 1 + else: + start_epoch = 0 + + return start_epoch diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/distributed.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/distributed.py" new file mode 100644 index 0000000..1ecc802 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/distributed.py" @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Distributed helpers.""" + +import functools +import logging +import pickle +import torch +import torch.distributed as dist + +_LOCAL_PROCESS_GROUP = None + + +def all_gather(tensors): + """ + All gathers the provided tensors from all processes across machines. + Args: + tensors (list): tensors to perform all gather across all processes in + all machines. + """ + + gather_list = [] + output_tensor = [] + world_size = dist.get_world_size() + for tensor in tensors: + tensor_placeholder = [ + torch.ones_like(tensor) for _ in range(world_size) + ] + dist.all_gather(tensor_placeholder, tensor, async_op=False) + gather_list.append(tensor_placeholder) + for gathered_tensor in gather_list: + output_tensor.append(torch.cat(gathered_tensor, dim=0)) + return output_tensor + + +def all_reduce(tensors, average=True): + """ + All reduce the provided tensors from all processes across machines. + Args: + tensors (list): tensors to perform all reduce across all processes in + all machines. + average (bool): scales the reduced tensor by the number of overall + processes across all machines. + """ + + for tensor in tensors: + dist.all_reduce(tensor, async_op=False) + if average: + world_size = dist.get_world_size() + for tensor in tensors: + tensor.mul_(1.0 / world_size) + return tensors + + +def init_process_group( + local_rank, + local_world_size, + shard_id, + num_shards, + init_method, + dist_backend="nccl", +): + """ + Initializes the default process group. + Args: + local_rank (int): the rank on the current local machine. + local_world_size (int): the world size (number of processes running) on + the current local machine. + shard_id (int): the shard index (machine rank) of the current machine. + num_shards (int): number of shards for distributed training. + init_method (string): supporting three different methods for + initializing process groups: + "file": use shared file system to initialize the groups across + different processes. + "tcp": use tcp address to initialize the groups across different + dist_backend (string): backend to use for distributed training. Options + includes gloo, mpi and nccl, the details can be found here: + https://pytorch.org/docs/stable/distributed.html + """ + # Sets the GPU to use. + torch.cuda.set_device(local_rank) + # Initialize the process group. + proc_rank = local_rank + shard_id * local_world_size + world_size = local_world_size * num_shards + dist.init_process_group( + backend=dist_backend, + init_method=init_method, + world_size=world_size, + rank=proc_rank, + ) + + +def is_master_proc(num_gpus=8): + """ + Determines if the current process is the master process. + """ + if torch.distributed.is_initialized(): + return dist.get_rank() % num_gpus == 0 + else: + return True + + +def is_root_proc(): + """ + Determines if the current process is the root process. + """ + if torch.distributed.is_initialized(): + return dist.get_rank() == 0 + else: + return True + + +def get_world_size(): + """ + Get the size of the world. + """ + if not dist.is_available(): + return 1 + if not dist.is_initialized(): + return 1 + return dist.get_world_size() + + +def get_rank(): + """ + Get the rank of the current process. + """ + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + return dist.get_rank() + + +def synchronize(): + """ + Helper function to synchronize (barrier) among all processes when + using distributed training + """ + if not dist.is_available(): + return + if not dist.is_initialized(): + return + world_size = dist.get_world_size() + if world_size == 1: + return + dist.barrier() + + +@functools.lru_cache() +def _get_global_gloo_group(): + """ + Return a process group based on gloo backend, containing all the ranks + The result is cached. + Returns: + (group): pytorch dist group. + """ + if dist.get_backend() == "nccl": + return dist.new_group(backend="gloo") + else: + return dist.group.WORLD + + +def _serialize_to_tensor(data, group): + """ + Seriialize the tensor to ByteTensor. Note that only `gloo` and `nccl` + backend is supported. + Args: + data (data): data to be serialized. + group (group): pytorch dist group. + Returns: + tensor (ByteTensor): tensor that serialized. + """ + + backend = dist.get_backend(group) + assert backend in ["gloo", "nccl"] + device = torch.device("cpu" if backend == "gloo" else "cuda") + + buffer = pickle.dumps(data) + if len(buffer) > 1024 ** 3: + logger = logging.getLogger(__name__) + logger.warning( + "Rank {} trying to all-gather {:.2f} GB of data on device {}".format( + get_rank(), len(buffer) / (1024 ** 3), device + ) + ) + storage = torch.ByteStorage.from_buffer(buffer) + tensor = torch.ByteTensor(storage).to(device=device) + return tensor + + +def _pad_to_largest_tensor(tensor, group): + """ + Padding all the tensors from different GPUs to the largest ones. + Args: + tensor (tensor): tensor to pad. + group (group): pytorch dist group. + Returns: + list[int]: size of the tensor, on each rank + Tensor: padded tensor that has the max size + """ + world_size = dist.get_world_size(group=group) + assert ( + world_size >= 1 + ), "comm.gather/all_gather must be called from ranks within the given group!" + local_size = torch.tensor( + [tensor.numel()], dtype=torch.int64, device=tensor.device + ) + size_list = [ + torch.zeros([1], dtype=torch.int64, device=tensor.device) + for _ in range(world_size) + ] + dist.all_gather(size_list, local_size, group=group) + size_list = [int(size.item()) for size in size_list] + + max_size = max(size_list) + + # we pad the tensor because torch all_gather does not support + # gathering tensors of different shapes + if local_size != max_size: + padding = torch.zeros( + (max_size - local_size,), dtype=torch.uint8, device=tensor.device + ) + tensor = torch.cat((tensor, padding), dim=0) + return size_list, tensor + + +def all_gather_unaligned(data, group=None): + """ + Run all_gather on arbitrary picklable data (not necessarily tensors). + + Args: + data: any picklable object + group: a torch process group. By default, will use a group which + contains all ranks on gloo backend. + + Returns: + list[data]: list of data gathered from each rank + """ + if get_world_size() == 1: + return [data] + if group is None: + group = _get_global_gloo_group() + if dist.get_world_size(group) == 1: + return [data] + + tensor = _serialize_to_tensor(data, group) + + size_list, tensor = _pad_to_largest_tensor(tensor, group) + max_size = max(size_list) + + # receiving Tensor from all ranks + tensor_list = [ + torch.empty((max_size,), dtype=torch.uint8, device=tensor.device) + for _ in size_list + ] + dist.all_gather(tensor_list, tensor, group=group) + + data_list = [] + for size, tensor in zip(size_list, tensor_list): + buffer = tensor.cpu().numpy().tobytes()[:size] + data_list.append(pickle.loads(buffer)) + + return data_list + + +def init_distributed_training(cfg): + """ + Initialize variables needed for distributed training. + """ + if cfg.NUM_GPUS <= 1: + return + num_gpus_per_machine = cfg.NUM_GPUS + num_machines = dist.get_world_size() // num_gpus_per_machine + for i in range(num_machines): + ranks_on_i = list( + range(i * num_gpus_per_machine, (i + 1) * num_gpus_per_machine) + ) + pg = dist.new_group(ranks_on_i) + if i == cfg.SHARD_ID: + global _LOCAL_PROCESS_GROUP + _LOCAL_PROCESS_GROUP = pg + + +def get_local_size() -> int: + """ + Returns: + The size of the per-machine process group, + i.e. the number of processes per machine. + """ + if not dist.is_available(): + return 1 + if not dist.is_initialized(): + return 1 + return dist.get_world_size(group=_LOCAL_PROCESS_GROUP) + + +def get_local_rank() -> int: + """ + Returns: + The rank of the current process within the local (per-machine) process group. + """ + if not dist.is_available(): + return 0 + if not dist.is_initialized(): + return 0 + assert _LOCAL_PROCESS_GROUP is not None + return dist.get_rank(group=_LOCAL_PROCESS_GROUP) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/env.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/env.py" new file mode 100644 index 0000000..2554915 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/env.py" @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Set up Environment.""" + +import slowfast.utils.logging as logging + +_ENV_SETUP_DONE = False + + +def setup_environment(): + global _ENV_SETUP_DONE + if _ENV_SETUP_DONE: + return + _ENV_SETUP_DONE = True diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/logging.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/logging.py" new file mode 100644 index 0000000..8a1bb65 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/logging.py" @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Logging.""" + +import atexit +import builtins +import decimal +import functools +import logging +import os +import sys +import simplejson +from iopath.common.file_io import g_pathmgr + +import slowfast.utils.distributed as du + + +def _suppress_print(): + """ + Suppresses printing from the current process. + """ + + def print_pass(*objects, sep=" ", end="\n", file=sys.stdout, flush=False): + pass + + builtins.print = print_pass + + +@functools.lru_cache(maxsize=None) +def _cached_log_stream(filename): + # Use 1K buffer if writing to cloud storage. + io = g_pathmgr.open( + filename, "a", buffering=1024 if "://" in filename else -1 + ) + atexit.register(io.close) + return io + + +def setup_logging(output_dir=None): + """ + Sets up the logging for multiple processes. Only enable the logging for the + master process, and suppress logging for the non-master processes. + """ + # Set up logging format. + _FORMAT = "[%(levelname)s: %(filename)s: %(lineno)4d]: %(message)s" + + if du.is_master_proc(): + # Enable logging for the master process. + logging.root.handlers = [] + else: + # Suppress logging for non-master processes. + _suppress_print() + + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + logger.propagate = False + plain_formatter = logging.Formatter( + "[%(asctime)s][%(levelname)s] %(filename)s: %(lineno)3d: %(message)s", + datefmt="%m/%d %H:%M:%S", + ) + + if du.is_master_proc(): + ch = logging.StreamHandler(stream=sys.stdout) + ch.setLevel(logging.DEBUG) + ch.setFormatter(plain_formatter) + logger.addHandler(ch) + + if output_dir is not None and du.is_master_proc(du.get_world_size()): + filename = os.path.join(output_dir, "stdout.log") + fh = logging.StreamHandler(_cached_log_stream(filename)) + fh.setLevel(logging.DEBUG) + fh.setFormatter(plain_formatter) + logger.addHandler(fh) + + +def get_logger(name): + """ + Retrieve the logger with the specified name or, if name is None, return a + logger which is the root logger of the hierarchy. + Args: + name (string): name of the logger. + """ + return logging.getLogger(name) + + +def log_json_stats(stats): + """ + Logs json stats. + Args: + stats (dict): a dictionary of statistical information to log. + """ + stats = { + k: decimal.Decimal("{:.5f}".format(v)) if isinstance(v, float) else v + for k, v in stats.items() + } + json_stats = simplejson.dumps(stats, sort_keys=True, use_decimal=True) + logger = get_logger(__name__) + logger.info("json_stats: {:s}".format(json_stats)) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/lr_policy.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/lr_policy.py" new file mode 100644 index 0000000..80cfdd4 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/lr_policy.py" @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Learning rate policy.""" + +import math + + +def get_lr_at_epoch(cfg, cur_epoch): + """ + Retrieve the learning rate of the current epoch with the option to perform + warm up in the beginning of the training stage. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + cur_epoch (float): the number of epoch of the current training stage. + """ + lr = get_lr_func(cfg.SOLVER.LR_POLICY)(cfg, cur_epoch) + # Perform warm up. + if cur_epoch < cfg.SOLVER.WARMUP_EPOCHS: + lr_start = cfg.SOLVER.WARMUP_START_LR + lr_end = get_lr_func(cfg.SOLVER.LR_POLICY)( + cfg, cfg.SOLVER.WARMUP_EPOCHS + ) + alpha = (lr_end - lr_start) / cfg.SOLVER.WARMUP_EPOCHS + lr = cur_epoch * alpha + lr_start + return lr + + +def lr_func_cosine(cfg, cur_epoch): + """ + Retrieve the learning rate to specified values at specified epoch with the + cosine learning rate schedule. Details can be found in: + Ilya Loshchilov, and Frank Hutter + SGDR: Stochastic Gradient Descent With Warm Restarts. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + cur_epoch (float): the number of epoch of the current training stage. + """ + offset = cfg.SOLVER.WARMUP_EPOCHS if cfg.SOLVER.COSINE_AFTER_WARMUP else 0.0 + assert cfg.SOLVER.COSINE_END_LR < cfg.SOLVER.BASE_LR + return ( + cfg.SOLVER.COSINE_END_LR + + (cfg.SOLVER.BASE_LR - cfg.SOLVER.COSINE_END_LR) + * ( + math.cos( + math.pi * (cur_epoch - offset) / (cfg.SOLVER.MAX_EPOCH - offset) + ) + + 1.0 + ) + * 0.5 + ) + + +def lr_func_steps_with_relative_lrs(cfg, cur_epoch): + """ + Retrieve the learning rate to specified values at specified epoch with the + steps with relative learning rate schedule. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + cur_epoch (float): the number of epoch of the current training stage. + """ + ind = get_step_index(cfg, cur_epoch) + return cfg.SOLVER.LRS[ind] * cfg.SOLVER.BASE_LR + + +def get_step_index(cfg, cur_epoch): + """ + Retrieves the lr step index for the given epoch. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + cur_epoch (float): the number of epoch of the current training stage. + """ + steps = cfg.SOLVER.STEPS + [cfg.SOLVER.MAX_EPOCH] + for ind, step in enumerate(steps): # NoQA + if cur_epoch < step: + break + return ind - 1 + + +def get_lr_func(lr_policy): + """ + Given the configs, retrieve the specified lr policy function. + Args: + lr_policy (string): the learning rate policy to use for the job. + """ + policy = "lr_func_" + lr_policy + if policy not in globals(): + raise NotImplementedError("Unknown LR policy: {}".format(lr_policy)) + else: + return globals()[policy] diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/meters.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/meters.py" new file mode 100644 index 0000000..bd5ee83 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/meters.py" @@ -0,0 +1,823 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Meters.""" + +import datetime +import numpy as np +import os +from collections import defaultdict, deque +import torch +from fvcore.common.timer import Timer +from sklearn.metrics import average_precision_score + +import slowfast.datasets.ava_helper as ava_helper +import slowfast.utils.logging as logging +import slowfast.utils.metrics as metrics +import slowfast.utils.misc as misc +from slowfast.utils.ava_eval_helper import ( + evaluate_ava, + read_csv, + read_exclusions, + read_labelmap, +) + +logger = logging.get_logger(__name__) + + +def get_ava_mini_groundtruth(full_groundtruth): + """ + Get the groundtruth annotations corresponding the "subset" of AVA val set. + We define the subset to be the frames such that (second % 4 == 0). + We optionally use subset for faster evaluation during training + (in order to track training progress). + Args: + full_groundtruth(dict): list of groundtruth. + """ + ret = [defaultdict(list), defaultdict(list), defaultdict(list)] + + for i in range(3): + for key in full_groundtruth[i].keys(): + if int(key.split(",")[1]) % 4 == 0: + ret[i][key] = full_groundtruth[i][key] + return ret + + +class AVAMeter(object): + """ + Measure the AVA train, val, and test stats. + """ + + def __init__(self, overall_iters, cfg, mode): + """ + overall_iters (int): the overall number of iterations of one epoch. + cfg (CfgNode): configs. + mode (str): `train`, `val`, or `test` mode. + """ + self.cfg = cfg + self.lr = None + self.loss = ScalarMeter(cfg.LOG_PERIOD) + self.full_ava_test = cfg.AVA.FULL_TEST_ON_VAL + self.mode = mode + self.iter_timer = Timer() + self.data_timer = Timer() + self.net_timer = Timer() + self.all_preds = [] + self.all_ori_boxes = [] + self.all_metadata = [] + self.overall_iters = overall_iters + self.excluded_keys = read_exclusions( + os.path.join(cfg.AVA.ANNOTATION_DIR, cfg.AVA.EXCLUSION_FILE) + ) + self.categories, self.class_whitelist = read_labelmap( + os.path.join(cfg.AVA.ANNOTATION_DIR, cfg.AVA.LABEL_MAP_FILE) + ) + gt_filename = os.path.join( + cfg.AVA.ANNOTATION_DIR, cfg.AVA.GROUNDTRUTH_FILE + ) + self.full_groundtruth = read_csv(gt_filename, self.class_whitelist) + self.mini_groundtruth = get_ava_mini_groundtruth(self.full_groundtruth) + + _, self.video_idx_to_name = ava_helper.load_image_lists( + cfg, mode == "train" + ) + self.output_dir = cfg.OUTPUT_DIR + + def log_iter_stats(self, cur_epoch, cur_iter): + """ + Log the stats. + Args: + cur_epoch (int): the current epoch. + cur_iter (int): the current iteration. + """ + + if (cur_iter + 1) % self.cfg.LOG_PERIOD != 0: + return + + eta_sec = self.iter_timer.seconds() * (self.overall_iters - cur_iter) + eta = str(datetime.timedelta(seconds=int(eta_sec))) + if self.mode == "train": + stats = { + "_type": "{}_iter".format(self.mode), + "cur_epoch": "{}".format(cur_epoch + 1), + "cur_iter": "{}".format(cur_iter + 1), + "eta": eta, + "dt": self.iter_timer.seconds(), + "dt_data": self.data_timer.seconds(), + "dt_net": self.net_timer.seconds(), + "mode": self.mode, + "loss": self.loss.get_win_median(), + "lr": self.lr, + } + elif self.mode == "val": + stats = { + "_type": "{}_iter".format(self.mode), + "cur_epoch": "{}".format(cur_epoch + 1), + "cur_iter": "{}".format(cur_iter + 1), + "eta": eta, + "dt": self.iter_timer.seconds(), + "dt_data": self.data_timer.seconds(), + "dt_net": self.net_timer.seconds(), + "mode": self.mode, + } + elif self.mode == "test": + stats = { + "_type": "{}_iter".format(self.mode), + "cur_iter": "{}".format(cur_iter + 1), + "eta": eta, + "dt": self.iter_timer.seconds(), + "dt_data": self.data_timer.seconds(), + "dt_net": self.net_timer.seconds(), + "mode": self.mode, + } + else: + raise NotImplementedError("Unknown mode: {}".format(self.mode)) + + logging.log_json_stats(stats) + + def iter_tic(self): + """ + Start to record time. + """ + self.iter_timer.reset() + self.data_timer.reset() + + def iter_toc(self): + """ + Stop to record time. + """ + self.iter_timer.pause() + self.net_timer.pause() + + def data_toc(self): + self.data_timer.pause() + self.net_timer.reset() + + def reset(self): + """ + Reset the Meter. + """ + self.loss.reset() + + self.all_preds = [] + self.all_ori_boxes = [] + self.all_metadata = [] + + def update_stats(self, preds, ori_boxes, metadata, loss=None, lr=None): + """ + Update the current stats. + Args: + preds (tensor): prediction embedding. + ori_boxes (tensor): original boxes (x1, y1, x2, y2). + metadata (tensor): metadata of the AVA data. + loss (float): loss value. + lr (float): learning rate. + """ + if self.mode in ["val", "test"]: + self.all_preds.append(preds) + self.all_ori_boxes.append(ori_boxes) + self.all_metadata.append(metadata) + if loss is not None: + self.loss.add_value(loss) + if lr is not None: + self.lr = lr + + def finalize_metrics(self, log=True): + """ + Calculate and log the final AVA metrics. + """ + all_preds = torch.cat(self.all_preds, dim=0) + all_ori_boxes = torch.cat(self.all_ori_boxes, dim=0) + all_metadata = torch.cat(self.all_metadata, dim=0) + + if self.mode == "test" or (self.full_ava_test and self.mode == "val"): + groundtruth = self.full_groundtruth + else: + groundtruth = self.mini_groundtruth + + self.full_map = evaluate_ava( + all_preds, + all_ori_boxes, + all_metadata.tolist(), + self.excluded_keys, + self.class_whitelist, + self.categories, + groundtruth=groundtruth, + video_idx_to_name=self.video_idx_to_name, + ) + if log: + stats = {"mode": self.mode, "map": self.full_map} + logging.log_json_stats(stats) + + def log_epoch_stats(self, cur_epoch): + """ + Log the stats of the current epoch. + Args: + cur_epoch (int): the number of current epoch. + """ + if self.mode in ["val", "test"]: + self.finalize_metrics(log=False) + stats = { + "_type": "{}_epoch".format(self.mode), + "cur_epoch": "{}".format(cur_epoch + 1), + "mode": self.mode, + "map": self.full_map, + "gpu_mem": "{:.2f}G".format(misc.gpu_mem_usage()), + "RAM": "{:.2f}/{:.2f}G".format(*misc.cpu_mem_usage()), + } + logging.log_json_stats(stats) + + +class TestMeter(object): + """ + Perform the multi-view ensemble for testing: each video with an unique index + will be sampled with multiple clips, and the predictions of the clips will + be aggregated to produce the final prediction for the video. + The accuracy is calculated with the given ground truth labels. + """ + + def __init__( + self, + num_videos, + num_clips, + num_cls, + overall_iters, + multi_label=False, + ensemble_method="sum", + ): + """ + Construct tensors to store the predictions and labels. Expect to get + num_clips predictions from each video, and calculate the metrics on + num_videos videos. + Args: + num_videos (int): number of videos to test. + num_clips (int): number of clips sampled from each video for + aggregating the final prediction for the video. + num_cls (int): number of classes for each prediction. + overall_iters (int): overall iterations for testing. + multi_label (bool): if True, use map as the metric. + ensemble_method (str): method to perform the ensemble, options + include "sum", and "max". + """ + + self.iter_timer = Timer() + self.data_timer = Timer() + self.net_timer = Timer() + self.num_clips = num_clips + self.overall_iters = overall_iters + self.multi_label = multi_label + self.ensemble_method = ensemble_method + # Initialize tensors. + self.video_preds = torch.zeros((num_videos, num_cls)) + if multi_label: + self.video_preds -= 1e10 + + self.video_labels = ( + torch.zeros((num_videos, num_cls)) + if multi_label + else torch.zeros((num_videos)).long() + ) + self.clip_count = torch.zeros((num_videos)).long() + self.topk_accs = [] + self.stats = {} + + # Reset metric. + self.reset() + + def reset(self): + """ + Reset the metric. + """ + self.clip_count.zero_() + self.video_preds.zero_() + if self.multi_label: + self.video_preds -= 1e10 + self.video_labels.zero_() + + def update_stats(self, preds, labels, clip_ids): + """ + Collect the predictions from the current batch and perform on-the-flight + summation as ensemble. + Args: + preds (tensor): predictions from the current batch. Dimension is + N x C where N is the batch size and C is the channel size + (num_cls). + labels (tensor): the corresponding labels of the current batch. + Dimension is N. + clip_ids (tensor): clip indexes of the current batch, dimension is + N. + """ + for ind in range(preds.shape[0]): + vid_id = int(clip_ids[ind]) // self.num_clips + if self.video_labels[vid_id].sum() > 0: + assert torch.equal( + self.video_labels[vid_id].type(torch.FloatTensor), + labels[ind].type(torch.FloatTensor), + ) + self.video_labels[vid_id] = labels[ind] + if self.ensemble_method == "sum": + self.video_preds[vid_id] += preds[ind] + elif self.ensemble_method == "max": + self.video_preds[vid_id] = torch.max( + self.video_preds[vid_id], preds[ind] + ) + else: + raise NotImplementedError( + "Ensemble Method {} is not supported".format( + self.ensemble_method + ) + ) + self.clip_count[vid_id] += 1 + + def log_iter_stats(self, cur_iter): + """ + Log the stats. + Args: + cur_iter (int): the current iteration of testing. + """ + eta_sec = self.iter_timer.seconds() * (self.overall_iters - cur_iter) + eta = str(datetime.timedelta(seconds=int(eta_sec))) + stats = { + "split": "test_iter", + "cur_iter": "{}".format(cur_iter + 1), + "eta": eta, + "time_diff": self.iter_timer.seconds(), + } + logging.log_json_stats(stats) + + def iter_tic(self): + """ + Start to record time. + """ + self.iter_timer.reset() + self.data_timer.reset() + + def iter_toc(self): + """ + Stop to record time. + """ + self.iter_timer.pause() + self.net_timer.pause() + + def data_toc(self): + self.data_timer.pause() + self.net_timer.reset() + + def finalize_metrics(self, ks=(1, 5)): + """ + Calculate and log the final ensembled metrics. + ks (tuple): list of top-k values for topk_accuracies. For example, + ks = (1, 5) correspods to top-1 and top-5 accuracy. + """ + if not all(self.clip_count == self.num_clips): + logger.warning( + "clip count {} ~= num clips {}".format( + ", ".join( + [ + "{}: {}".format(i, k) + for i, k in enumerate(self.clip_count.tolist()) + ] + ), + self.num_clips, + ) + ) + + self.stats = {"split": "test_final"} + if self.multi_label: + map = get_map( + self.video_preds.cpu().numpy(), self.video_labels.cpu().numpy() + ) + self.stats["map"] = map + else: + num_topks_correct = metrics.topks_correct( + self.video_preds, self.video_labels, ks + ) + topks = [ + (x / self.video_preds.size(0)) * 100.0 + for x in num_topks_correct + ] + assert len({len(ks), len(topks)}) == 1 + for k, topk in zip(ks, topks): + self.stats["top{}_acc".format(k)] = "{:.{prec}f}".format( + topk, prec=2 + ) + logging.log_json_stats(self.stats) + + +class ScalarMeter(object): + """ + A scalar meter uses a deque to track a series of scaler values with a given + window size. It supports calculating the median and average values of the + window, and also supports calculating the global average. + """ + + def __init__(self, window_size): + """ + Args: + window_size (int): size of the max length of the deque. + """ + self.deque = deque(maxlen=window_size) + self.total = 0.0 + self.count = 0 + + def reset(self): + """ + Reset the deque. + """ + self.deque.clear() + self.total = 0.0 + self.count = 0 + + def add_value(self, value): + """ + Add a new scalar value to the deque. + """ + self.deque.append(value) + self.count += 1 + self.total += value + + def get_win_median(self): + """ + Calculate the current median value of the deque. + """ + return np.median(self.deque) + + def get_win_avg(self): + """ + Calculate the current average value of the deque. + """ + return np.mean(self.deque) + + def get_global_avg(self): + """ + Calculate the global mean value. + """ + return self.total / self.count + + +class TrainMeter(object): + """ + Measure training stats. + """ + + def __init__(self, epoch_iters, cfg): + """ + Args: + epoch_iters (int): the overall number of iterations of one epoch. + cfg (CfgNode): configs. + """ + self._cfg = cfg + self.epoch_iters = epoch_iters + self.MAX_EPOCH = cfg.SOLVER.MAX_EPOCH * epoch_iters + self.iter_timer = Timer() + self.data_timer = Timer() + self.net_timer = Timer() + self.loss = ScalarMeter(cfg.LOG_PERIOD) + self.loss_total = 0.0 + self.lr = None + # Current minibatch errors (smoothed over a window). + self.mb_top1_err = ScalarMeter(cfg.LOG_PERIOD) + self.mb_top5_err = ScalarMeter(cfg.LOG_PERIOD) + # Number of misclassified examples. + self.num_top1_mis = 0 + self.num_top5_mis = 0 + self.num_samples = 0 + self.output_dir = cfg.OUTPUT_DIR + + def reset(self): + """ + Reset the Meter. + """ + self.loss.reset() + self.loss_total = 0.0 + self.lr = None + self.mb_top1_err.reset() + self.mb_top5_err.reset() + self.num_top1_mis = 0 + self.num_top5_mis = 0 + self.num_samples = 0 + + def iter_tic(self): + """ + Start to record time. + """ + self.iter_timer.reset() + self.data_timer.reset() + + def iter_toc(self): + """ + Stop to record time. + """ + self.iter_timer.pause() + self.net_timer.pause() + + def data_toc(self): + self.data_timer.pause() + self.net_timer.reset() + + def update_stats(self, top1_err, top5_err, loss, lr, mb_size): + """ + Update the current stats. + Args: + top1_err (float): top1 error rate. + top5_err (float): top5 error rate. + loss (float): loss value. + lr (float): learning rate. + mb_size (int): mini batch size. + """ + self.loss.add_value(loss) + self.lr = lr + self.loss_total += loss * mb_size + self.num_samples += mb_size + + if not self._cfg.DATA.MULTI_LABEL: + # Current minibatch stats + self.mb_top1_err.add_value(top1_err) + self.mb_top5_err.add_value(top5_err) + # Aggregate stats + self.num_top1_mis += top1_err * mb_size + self.num_top5_mis += top5_err * mb_size + + def log_iter_stats(self, cur_epoch, cur_iter): + """ + log the stats of the current iteration. + Args: + cur_epoch (int): the number of current epoch. + cur_iter (int): the number of current iteration. + """ + if (cur_iter + 1) % self._cfg.LOG_PERIOD != 0: + return + eta_sec = self.iter_timer.seconds() * ( + self.MAX_EPOCH - (cur_epoch * self.epoch_iters + cur_iter + 1) + ) + eta = str(datetime.timedelta(seconds=int(eta_sec))) + stats = { + "_type": "train_iter", + "epoch": "{}/{}".format(cur_epoch + 1, self._cfg.SOLVER.MAX_EPOCH), + "iter": "{}/{}".format(cur_iter + 1, self.epoch_iters), + "dt": self.iter_timer.seconds(), + "dt_data": self.data_timer.seconds(), + "dt_net": self.net_timer.seconds(), + "eta": eta, + "loss": self.loss.get_win_median(), + "lr": self.lr, + "gpu_mem": "{:.2f}G".format(misc.gpu_mem_usage()), + } + if not self._cfg.DATA.MULTI_LABEL: + stats["top1_err"] = self.mb_top1_err.get_win_median() + stats["top5_err"] = self.mb_top5_err.get_win_median() + logging.log_json_stats(stats) + + def log_epoch_stats(self, cur_epoch): + """ + Log the stats of the current epoch. + Args: + cur_epoch (int): the number of current epoch. + """ + eta_sec = self.iter_timer.seconds() * ( + self.MAX_EPOCH - (cur_epoch + 1) * self.epoch_iters + ) + eta = str(datetime.timedelta(seconds=int(eta_sec))) + stats = { + "_type": "train_epoch", + "epoch": "{}/{}".format(cur_epoch + 1, self._cfg.SOLVER.MAX_EPOCH), + "dt": self.iter_timer.seconds(), + "dt_data": self.data_timer.seconds(), + "dt_net": self.net_timer.seconds(), + "eta": eta, + "lr": self.lr, + "gpu_mem": "{:.2f}G".format(misc.gpu_mem_usage()), + "RAM": "{:.2f}/{:.2f}G".format(*misc.cpu_mem_usage()), + } + if not self._cfg.DATA.MULTI_LABEL: + top1_err = self.num_top1_mis / self.num_samples + top5_err = self.num_top5_mis / self.num_samples + avg_loss = self.loss_total / self.num_samples + stats["top1_err"] = top1_err + stats["top5_err"] = top5_err + stats["loss"] = avg_loss + logging.log_json_stats(stats) + + +class ValMeter(object): + """ + Measures validation stats. + """ + + def __init__(self, max_iter, cfg): + """ + Args: + max_iter (int): the max number of iteration of the current epoch. + cfg (CfgNode): configs. + """ + self._cfg = cfg + self.max_iter = max_iter + self.iter_timer = Timer() + self.data_timer = Timer() + self.net_timer = Timer() + # Current minibatch errors (smoothed over a window). + self.mb_top1_err = ScalarMeter(cfg.LOG_PERIOD) + self.mb_top5_err = ScalarMeter(cfg.LOG_PERIOD) + # Min errors (over the full val set). + self.min_top1_err = 100.0 + self.min_top5_err = 100.0 + # Number of misclassified examples. + self.num_top1_mis = 0 + self.num_top5_mis = 0 + self.num_samples = 0 + self.all_preds = [] + self.all_labels = [] + self.output_dir = cfg.OUTPUT_DIR + + def reset(self): + """ + Reset the Meter. + """ + self.iter_timer.reset() + self.mb_top1_err.reset() + self.mb_top5_err.reset() + self.num_top1_mis = 0 + self.num_top5_mis = 0 + self.num_samples = 0 + self.all_preds = [] + self.all_labels = [] + + def iter_tic(self): + """ + Start to record time. + """ + self.iter_timer.reset() + self.data_timer.reset() + + def iter_toc(self): + """ + Stop to record time. + """ + self.iter_timer.pause() + self.net_timer.pause() + + def data_toc(self): + self.data_timer.pause() + self.net_timer.reset() + + def update_stats(self, top1_err, top5_err, mb_size): + """ + Update the current stats. + Args: + top1_err (float): top1 error rate. + top5_err (float): top5 error rate. + mb_size (int): mini batch size. + """ + self.mb_top1_err.add_value(top1_err) + self.mb_top5_err.add_value(top5_err) + self.num_top1_mis += top1_err * mb_size + self.num_top5_mis += top5_err * mb_size + self.num_samples += mb_size + + def update_predictions(self, preds, labels): + """ + Update predictions and labels. + Args: + preds (tensor): model output predictions. + labels (tensor): labels. + """ + # TODO: merge update_prediction with update_stats. + self.all_preds.append(preds) + self.all_labels.append(labels) + + def log_iter_stats(self, cur_epoch, cur_iter): + """ + log the stats of the current iteration. + Args: + cur_epoch (int): the number of current epoch. + cur_iter (int): the number of current iteration. + """ + if (cur_iter + 1) % self._cfg.LOG_PERIOD != 0: + return + eta_sec = self.iter_timer.seconds() * (self.max_iter - cur_iter - 1) + eta = str(datetime.timedelta(seconds=int(eta_sec))) + stats = { + "_type": "val_iter", + "epoch": "{}/{}".format(cur_epoch + 1, self._cfg.SOLVER.MAX_EPOCH), + "iter": "{}/{}".format(cur_iter + 1, self.max_iter), + "time_diff": self.iter_timer.seconds(), + "eta": eta, + "gpu_mem": "{:.2f}G".format(misc.gpu_mem_usage()), + } + if not self._cfg.DATA.MULTI_LABEL: + stats["top1_err"] = self.mb_top1_err.get_win_median() + stats["top5_err"] = self.mb_top5_err.get_win_median() + logging.log_json_stats(stats) + + def log_epoch_stats(self, cur_epoch): + """ + Log the stats of the current epoch. + Args: + cur_epoch (int): the number of current epoch. + """ + stats = { + "_type": "val_epoch", + "epoch": "{}/{}".format(cur_epoch + 1, self._cfg.SOLVER.MAX_EPOCH), + "time_diff": self.iter_timer.seconds(), + "gpu_mem": "{:.2f}G".format(misc.gpu_mem_usage()), + "RAM": "{:.2f}/{:.2f}G".format(*misc.cpu_mem_usage()), + } + if self._cfg.DATA.MULTI_LABEL: + stats["map"] = get_map( + torch.cat(self.all_preds).cpu().numpy(), + torch.cat(self.all_labels).cpu().numpy(), + ) + else: + top1_err = self.num_top1_mis / self.num_samples + top5_err = self.num_top5_mis / self.num_samples + self.min_top1_err = min(self.min_top1_err, top1_err) + self.min_top5_err = min(self.min_top5_err, top5_err) + + stats["top1_err"] = top1_err + stats["top5_err"] = top5_err + stats["min_top1_err"] = self.min_top1_err + stats["min_top5_err"] = self.min_top5_err + + logging.log_json_stats(stats) + + +def get_map(preds, labels): + """ + Compute mAP for multi-label case. + Args: + preds (numpy tensor): num_examples x num_classes. + labels (numpy tensor): num_examples x num_classes. + Returns: + mean_ap (int): final mAP score. + """ + + logger.info("Getting mAP for {} examples".format(preds.shape[0])) + + preds = preds[:, ~(np.all(labels == 0, axis=0))] + labels = labels[:, ~(np.all(labels == 0, axis=0))] + aps = [0] + try: + aps = average_precision_score(labels, preds, average=None) + except ValueError: + print( + "Average precision requires a sufficient number of samples \ + in a batch which are missing in this sample." + ) + + mean_ap = np.mean(aps) + return mean_ap + + +class EpochTimer: + """ + A timer which computes the epoch time. + """ + + def __init__(self) -> None: + self.timer = Timer() + self.timer.reset() + self.epoch_times = [] + + def reset(self) -> None: + """ + Reset the epoch timer. + """ + self.timer.reset() + self.epoch_times = [] + + def epoch_tic(self): + """ + Start to record time. + """ + self.timer.reset() + + def epoch_toc(self): + """ + Stop to record time. + """ + self.timer.pause() + self.epoch_times.append(self.timer.seconds()) + + def last_epoch_time(self): + """ + Get the time for the last epoch. + """ + assert len(self.epoch_times) > 0, "No epoch time has been recorded!" + + return self.epoch_times[-1] + + def avg_epoch_time(self): + """ + Calculate the average epoch time among the recorded epochs. + """ + assert len(self.epoch_times) > 0, "No epoch time has been recorded!" + + return np.mean(self.epoch_times) + + def median_epoch_time(self): + """ + Calculate the median epoch time among the recorded epochs. + """ + assert len(self.epoch_times) > 0, "No epoch time has been recorded!" + + return np.median(self.epoch_times) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/metrics.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/metrics.py" new file mode 100644 index 0000000..c1303cc --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/metrics.py" @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Functions for computing metrics.""" + +import torch + + +def topks_correct(preds, labels, ks): + """ + Given the predictions, labels, and a list of top-k values, compute the + number of correct predictions for each top-k value. + + Args: + preds (array): array of predictions. Dimension is batchsize + N x ClassNum. + labels (array): array of labels. Dimension is batchsize N. + ks (list): list of top-k values. For example, ks = [1, 5] correspods + to top-1 and top-5. + + Returns: + topks_correct (list): list of numbers, where the `i`-th entry + corresponds to the number of top-`ks[i]` correct predictions. + """ + assert preds.size(0) == labels.size( + 0 + ), "Batch dim of predictions and labels must match" + # Find the top max_k predictions for each sample + _top_max_k_vals, top_max_k_inds = torch.topk( + preds, max(ks), dim=1, largest=True, sorted=True + ) + # (batch_size, max_k) -> (max_k, batch_size). + top_max_k_inds = top_max_k_inds.t() + # (batch_size, ) -> (max_k, batch_size). + rep_max_k_labels = labels.view(1, -1).expand_as(top_max_k_inds) + # (i, j) = 1 if top i-th prediction for the j-th sample is correct. + top_max_k_correct = top_max_k_inds.eq(rep_max_k_labels) + # Compute the number of topk correct predictions for each k. + topks_correct = [top_max_k_correct[:k, :].float().sum() for k in ks] + return topks_correct + + +def topk_errors(preds, labels, ks): + """ + Computes the top-k error for each k. + Args: + preds (array): array of predictions. Dimension is N. + labels (array): array of labels. Dimension is N. + ks (list): list of ks to calculate the top accuracies. + """ + num_topks_correct = topks_correct(preds, labels, ks) + return [(1.0 - x / preds.size(0)) * 100.0 for x in num_topks_correct] + + +def topk_accuracies(preds, labels, ks): + """ + Computes the top-k accuracy for each k. + Args: + preds (array): array of predictions. Dimension is N. + labels (array): array of labels. Dimension is N. + ks (list): list of ks to calculate the top accuracies. + """ + num_topks_correct = topks_correct(preds, labels, ks) + return [(x / preds.size(0)) * 100.0 for x in num_topks_correct] diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/misc.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/misc.py" new file mode 100644 index 0000000..46e9dea --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/misc.py" @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import json +import logging +import math +import numpy as np +import os +from datetime import datetime +import psutil +import torch +from fvcore.nn.activation_count import activation_count +from fvcore.nn.flop_count import flop_count +from iopath.common.file_io import g_pathmgr +from matplotlib import pyplot as plt +from torch import nn + +import slowfast.utils.logging as logging +import slowfast.utils.multiprocessing as mpu +from slowfast.datasets.utils import pack_pathway_output +from slowfast.models.batchnorm_helper import SubBatchNorm3d + +logger = logging.get_logger(__name__) + + +def check_nan_losses(loss): + """ + Determine whether the loss is NaN (not a number). + Args: + loss (loss): loss to check whether is NaN. + """ + if math.isnan(loss): + raise RuntimeError("ERROR: Got NaN losses {}".format(datetime.now())) + + +def params_count(model, ignore_bn=False): + """ + Compute the number of parameters. + Args: + model (model): model to count the number of parameters. + """ + if not ignore_bn: + return np.sum([p.numel() for p in model.parameters()]).item() + else: + count = 0 + for m in model.modules(): + if not isinstance(m, nn.BatchNorm3d): + for p in m.parameters(recurse=False): + count += p.numel() + return count + + +def gpu_mem_usage(): + """ + Compute the GPU memory usage for the current device (GB). + """ + if torch.cuda.is_available(): + mem_usage_bytes = torch.cuda.max_memory_allocated() + else: + mem_usage_bytes = 0 + return mem_usage_bytes / 1024 ** 3 + + +def cpu_mem_usage(): + """ + Compute the system memory (RAM) usage for the current device (GB). + Returns: + usage (float): used memory (GB). + total (float): total memory (GB). + """ + vram = psutil.virtual_memory() + usage = (vram.total - vram.available) / 1024 ** 3 + total = vram.total / 1024 ** 3 + + return usage, total + + +def _get_model_analysis_input(cfg, use_train_input): + """ + Return a dummy input for model analysis with batch size 1. The input is + used for analyzing the model (counting flops and activations etc.). + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + use_train_input (bool): if True, return the input for training. Otherwise, + return the input for testing. + + Returns: + inputs: the input for model analysis. + """ + rgb_dimension = 3 + if use_train_input: + if cfg.TRAIN.DATASET in ["imagenet", "imagenetprefetch"]: + input_tensors = torch.rand( + rgb_dimension, + cfg.DATA.TRAIN_CROP_SIZE, + cfg.DATA.TRAIN_CROP_SIZE, + ) + else: + input_tensors = torch.rand( + rgb_dimension, + cfg.DATA.NUM_FRAMES, + cfg.DATA.TRAIN_CROP_SIZE, + cfg.DATA.TRAIN_CROP_SIZE, + ) + else: + if cfg.TEST.DATASET in ["imagenet", "imagenetprefetch"]: + input_tensors = torch.rand( + rgb_dimension, + cfg.DATA.TEST_CROP_SIZE, + cfg.DATA.TEST_CROP_SIZE, + ) + else: + input_tensors = torch.rand( + rgb_dimension, + cfg.DATA.NUM_FRAMES, + cfg.DATA.TEST_CROP_SIZE, + cfg.DATA.TEST_CROP_SIZE, + ) + model_inputs = pack_pathway_output(cfg, input_tensors) + for i in range(len(model_inputs)): + model_inputs[i] = model_inputs[i].unsqueeze(0) + if cfg.NUM_GPUS: + model_inputs[i] = model_inputs[i].cuda(non_blocking=True) + + # If detection is enabled, count flops for one proposal. + if cfg.DETECTION.ENABLE: + bbox = torch.tensor([[0, 0, 1.0, 0, 1.0]]) + if cfg.NUM_GPUS: + bbox = bbox.cuda() + inputs = (model_inputs, bbox) + else: + inputs = (model_inputs,) + return inputs + + +def get_model_stats(model, cfg, mode, use_train_input): + """ + Compute statistics for the current model given the config. + Args: + model (model): model to perform analysis. + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + mode (str): Options include `flop` or `activation`. Compute either flop + (gflops) or activation count (mega). + use_train_input (bool): if True, compute statistics for training. Otherwise, + compute statistics for testing. + + Returns: + float: the total number of count of the given model. + """ + assert mode in [ + "flop", + "activation", + ], "'{}' not supported for model analysis".format(mode) + if mode == "flop": + model_stats_fun = flop_count + elif mode == "activation": + model_stats_fun = activation_count + + # Set model to evaluation mode for analysis. + # Evaluation mode can avoid getting stuck with sync batchnorm. + model_mode = model.training + model.eval() + inputs = _get_model_analysis_input(cfg, use_train_input) + count_dict, *_ = model_stats_fun(model, inputs) + count = sum(count_dict.values()) + model.train(model_mode) + return count + + +def log_model_info(model, cfg, use_train_input=True): + """ + Log info, includes number of parameters, gpu usage, gflops and activation count. + The model info is computed when the model is in validation mode. + Args: + model (model): model to log the info. + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + use_train_input (bool): if True, log info for training. Otherwise, + log info for testing. + """ + logger.info("Model:\n{}".format(model)) + logger.info("Params: {:,}".format(params_count(model))) + logger.info("Mem: {:,} MB".format(gpu_mem_usage())) + logger.info( + "Flops: {:,} G".format( + get_model_stats(model, cfg, "flop", use_train_input) + ) + ) + logger.info( + "Activations: {:,} M".format( + get_model_stats(model, cfg, "activation", use_train_input) + ) + ) + logger.info("nvidia-smi") + os.system("nvidia-smi") + + +def is_eval_epoch(cfg, cur_epoch, multigrid_schedule): + """ + Determine if the model should be evaluated at the current epoch. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + cur_epoch (int): current epoch. + multigrid_schedule (List): schedule for multigrid training. + """ + if cur_epoch + 1 == cfg.SOLVER.MAX_EPOCH: + return True + if multigrid_schedule is not None: + prev_epoch = 0 + for s in multigrid_schedule: + if cur_epoch < s[-1]: + period = max( + (s[-1] - prev_epoch) // cfg.MULTIGRID.EVAL_FREQ + 1, 1 + ) + return (s[-1] - 1 - cur_epoch) % period == 0 + prev_epoch = s[-1] + + return (cur_epoch + 1) % cfg.TRAIN.EVAL_PERIOD == 0 + + +def plot_input(tensor, bboxes=(), texts=(), path="./tmp_vis.png"): + """ + Plot the input tensor with the optional bounding box and save it to disk. + Args: + tensor (tensor): a tensor with shape of `NxCxHxW`. + bboxes (tuple): bounding boxes with format of [[x, y, h, w]]. + texts (tuple): a tuple of string to plot. + path (str): path to the image to save to. + """ + tensor = tensor.float() + tensor = tensor - tensor.min() + tensor = tensor / tensor.max() + f, ax = plt.subplots(nrows=1, ncols=tensor.shape[0], figsize=(50, 20)) + for i in range(tensor.shape[0]): + ax[i].axis("off") + ax[i].imshow(tensor[i].permute(1, 2, 0)) + # ax[1][0].axis('off') + if bboxes is not None and len(bboxes) > i: + for box in bboxes[i]: + x1, y1, x2, y2 = box + ax[i].vlines(x1, y1, y2, colors="g", linestyles="solid") + ax[i].vlines(x2, y1, y2, colors="g", linestyles="solid") + ax[i].hlines(y1, x1, x2, colors="g", linestyles="solid") + ax[i].hlines(y2, x1, x2, colors="g", linestyles="solid") + + if texts is not None and len(texts) > i: + ax[i].text(0, 0, texts[i]) + f.savefig(path) + + +def frozen_bn_stats(model): + """ + Set all the bn layers to eval mode. + Args: + model (model): model to set bn layers to eval mode. + """ + for m in model.modules(): + if isinstance(m, nn.BatchNorm3d): + m.eval() + + +def aggregate_sub_bn_stats(module): + """ + Recursively find all SubBN modules and aggregate sub-BN stats. + Args: + module (nn.Module) + Returns: + count (int): number of SubBN module found. + """ + count = 0 + for child in module.children(): + if isinstance(child, SubBatchNorm3d): + child.aggregate_stats() + count += 1 + else: + count += aggregate_sub_bn_stats(child) + return count + + +def launch_job(cfg, init_method, func, daemon=False): + """ + Run 'func' on one or more GPUs, specified in cfg + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + init_method (str): initialization method to launch the job with multiple + devices. + func (function): job to run on GPU(s) + daemon (bool): The spawned processes’ daemon flag. If set to True, + daemonic processes will be created + """ + if cfg.NUM_GPUS > 1: + torch.multiprocessing.spawn( + mpu.run, + nprocs=cfg.NUM_GPUS, + args=( + cfg.NUM_GPUS, + func, + init_method, + cfg.SHARD_ID, + cfg.NUM_SHARDS, + cfg.DIST_BACKEND, + cfg, + ), + daemon=daemon, + ) + else: + func(cfg=cfg) + + +def get_class_names(path, parent_path=None, subset_path=None): + """ + Read json file with entries {classname: index} and return + an array of class names in order. + If parent_path is provided, load and map all children to their ids. + Args: + path (str): path to class ids json file. + File must be in the format {"class1": id1, "class2": id2, ...} + parent_path (Optional[str]): path to parent-child json file. + File must be in the format {"parent1": ["child1", "child2", ...], ...} + subset_path (Optional[str]): path to text file containing a subset + of class names, separated by newline characters. + Returns: + class_names (list of strs): list of class names. + class_parents (dict): a dictionary where key is the name of the parent class + and value is a list of ids of the children classes. + subset_ids (list of ints): list of ids of the classes provided in the + subset file. + """ + try: + with g_pathmgr.open(path, "r") as f: + class2idx = json.load(f) + except Exception as err: + print("Fail to load file from {} with error {}".format(path, err)) + return + + max_key = max(class2idx.values()) + class_names = [None] * (max_key + 1) + + for k, i in class2idx.items(): + class_names[i] = k + + class_parent = None + if parent_path is not None and parent_path != "": + try: + with g_pathmgr.open(parent_path, "r") as f: + d_parent = json.load(f) + except EnvironmentError as err: + print( + "Fail to load file from {} with error {}".format( + parent_path, err + ) + ) + return + class_parent = {} + for parent, children in d_parent.items(): + indices = [ + class2idx[c] for c in children if class2idx.get(c) is not None + ] + class_parent[parent] = indices + + subset_ids = None + if subset_path is not None and subset_path != "": + try: + with g_pathmgr.open(subset_path, "r") as f: + subset = f.read().split("\n") + subset_ids = [ + class2idx[name] + for name in subset + if class2idx.get(name) is not None + ] + except EnvironmentError as err: + print( + "Fail to load file from {} with error {}".format( + subset_path, err + ) + ) + return + + return class_names, class_parent, subset_ids diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/multigrid.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/multigrid.py" new file mode 100644 index 0000000..4aed24b --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/multigrid.py" @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Helper functions for multigrid training.""" + +import numpy as np + +import slowfast.utils.logging as logging + +logger = logging.get_logger(__name__) + + +class MultigridSchedule(object): + """ + This class defines multigrid training schedule and update cfg accordingly. + """ + + def init_multigrid(self, cfg): + """ + Update cfg based on multigrid settings. + Args: + cfg (configs): configs that contains training and multigrid specific + hyperparameters. Details can be seen in + slowfast/config/defaults.py. + Returns: + cfg (configs): the updated cfg. + """ + self.schedule = None + # We may modify cfg.TRAIN.BATCH_SIZE, cfg.DATA.NUM_FRAMES, and + # cfg.DATA.TRAIN_CROP_SIZE during training, so we store their original + # value in cfg and use them as global variables. + cfg.MULTIGRID.DEFAULT_B = cfg.TRAIN.BATCH_SIZE + cfg.MULTIGRID.DEFAULT_T = cfg.DATA.NUM_FRAMES + cfg.MULTIGRID.DEFAULT_S = cfg.DATA.TRAIN_CROP_SIZE + + if cfg.MULTIGRID.LONG_CYCLE: + self.schedule = self.get_long_cycle_schedule(cfg) + cfg.SOLVER.STEPS = [0] + [s[-1] for s in self.schedule] + # Fine-tuning phase. + cfg.SOLVER.STEPS[-1] = ( + cfg.SOLVER.STEPS[-2] + cfg.SOLVER.STEPS[-1] + ) // 2 + cfg.SOLVER.LRS = [ + cfg.SOLVER.GAMMA ** s[0] * s[1][0] for s in self.schedule + ] + # Fine-tuning phase. + cfg.SOLVER.LRS = cfg.SOLVER.LRS[:-1] + [ + cfg.SOLVER.LRS[-2], + cfg.SOLVER.LRS[-1], + ] + + cfg.SOLVER.MAX_EPOCH = self.schedule[-1][-1] + + elif cfg.MULTIGRID.SHORT_CYCLE: + cfg.SOLVER.STEPS = [ + int(s * cfg.MULTIGRID.EPOCH_FACTOR) for s in cfg.SOLVER.STEPS + ] + cfg.SOLVER.MAX_EPOCH = int( + cfg.SOLVER.MAX_EPOCH * cfg.MULTIGRID.EPOCH_FACTOR + ) + return cfg + + def update_long_cycle(self, cfg, cur_epoch): + """ + Before every epoch, check if long cycle shape should change. If it + should, update cfg accordingly. + Args: + cfg (configs): configs that contains training and multigrid specific + hyperparameters. Details can be seen in + slowfast/config/defaults.py. + cur_epoch (int): current epoch index. + Returns: + cfg (configs): the updated cfg. + changed (bool): do we change long cycle shape at this epoch? + """ + base_b, base_t, base_s = get_current_long_cycle_shape( + self.schedule, cur_epoch + ) + if base_s != cfg.DATA.TRAIN_CROP_SIZE or base_t != cfg.DATA.NUM_FRAMES: + + cfg.DATA.NUM_FRAMES = base_t + cfg.DATA.TRAIN_CROP_SIZE = base_s + cfg.TRAIN.BATCH_SIZE = base_b * cfg.MULTIGRID.DEFAULT_B + + bs_factor = ( + float(cfg.TRAIN.BATCH_SIZE / cfg.NUM_GPUS) + / cfg.MULTIGRID.BN_BASE_SIZE + ) + + if bs_factor < 1: + cfg.BN.NORM_TYPE = "sync_batchnorm" + cfg.BN.NUM_SYNC_DEVICES = int(1.0 / bs_factor) + elif bs_factor > 1: + cfg.BN.NORM_TYPE = "sub_batchnorm" + cfg.BN.NUM_SPLITS = int(bs_factor) + else: + cfg.BN.NORM_TYPE = "batchnorm" + + cfg.MULTIGRID.LONG_CYCLE_SAMPLING_RATE = cfg.DATA.SAMPLING_RATE * ( + cfg.MULTIGRID.DEFAULT_T // cfg.DATA.NUM_FRAMES + ) + logger.info("Long cycle updates:") + logger.info("\tBN.NORM_TYPE: {}".format(cfg.BN.NORM_TYPE)) + if cfg.BN.NORM_TYPE == "sync_batchnorm": + logger.info( + "\tBN.NUM_SYNC_DEVICES: {}".format(cfg.BN.NUM_SYNC_DEVICES) + ) + elif cfg.BN.NORM_TYPE == "sub_batchnorm": + logger.info("\tBN.NUM_SPLITS: {}".format(cfg.BN.NUM_SPLITS)) + logger.info("\tTRAIN.BATCH_SIZE: {}".format(cfg.TRAIN.BATCH_SIZE)) + logger.info( + "\tDATA.NUM_FRAMES x LONG_CYCLE_SAMPLING_RATE: {}x{}".format( + cfg.DATA.NUM_FRAMES, cfg.MULTIGRID.LONG_CYCLE_SAMPLING_RATE + ) + ) + logger.info( + "\tDATA.TRAIN_CROP_SIZE: {}".format(cfg.DATA.TRAIN_CROP_SIZE) + ) + return cfg, True + else: + return cfg, False + + def get_long_cycle_schedule(self, cfg): + """ + Based on multigrid hyperparameters, define the schedule of a long cycle. + Args: + cfg (configs): configs that contains training and multigrid specific + hyperparameters. Details can be seen in + slowfast/config/defaults.py. + Returns: + schedule (list): Specifies a list long cycle base shapes and their + corresponding training epochs. + """ + + steps = cfg.SOLVER.STEPS + + default_size = float( + cfg.DATA.NUM_FRAMES * cfg.DATA.TRAIN_CROP_SIZE ** 2 + ) + default_iters = steps[-1] + + # Get shapes and average batch size for each long cycle shape. + avg_bs = [] + all_shapes = [] + for t_factor, s_factor in cfg.MULTIGRID.LONG_CYCLE_FACTORS: + base_t = int(round(cfg.DATA.NUM_FRAMES * t_factor)) + base_s = int(round(cfg.DATA.TRAIN_CROP_SIZE * s_factor)) + if cfg.MULTIGRID.SHORT_CYCLE: + shapes = [ + [ + base_t, + cfg.MULTIGRID.DEFAULT_S + * cfg.MULTIGRID.SHORT_CYCLE_FACTORS[0], + ], + [ + base_t, + cfg.MULTIGRID.DEFAULT_S + * cfg.MULTIGRID.SHORT_CYCLE_FACTORS[1], + ], + [base_t, base_s], + ] + else: + shapes = [[base_t, base_s]] + + # (T, S) -> (B, T, S) + shapes = [ + [int(round(default_size / (s[0] * s[1] * s[1]))), s[0], s[1]] + for s in shapes + ] + avg_bs.append(np.mean([s[0] for s in shapes])) + all_shapes.append(shapes) + + # Get schedule regardless of cfg.MULTIGRID.EPOCH_FACTOR. + total_iters = 0 + schedule = [] + for step_index in range(len(steps) - 1): + step_epochs = steps[step_index + 1] - steps[step_index] + + for long_cycle_index, shapes in enumerate(all_shapes): + cur_epochs = ( + step_epochs * avg_bs[long_cycle_index] / sum(avg_bs) + ) + + cur_iters = cur_epochs / avg_bs[long_cycle_index] + total_iters += cur_iters + schedule.append((step_index, shapes[-1], cur_epochs)) + + iter_saving = default_iters / total_iters + + final_step_epochs = cfg.SOLVER.MAX_EPOCH - steps[-1] + + # We define the fine-tuning phase to have the same amount of iteration + # saving as the rest of the training. + ft_epochs = final_step_epochs / iter_saving * avg_bs[-1] + + schedule.append((step_index + 1, all_shapes[-1][2], ft_epochs)) + + # Obtrain final schedule given desired cfg.MULTIGRID.EPOCH_FACTOR. + x = ( + cfg.SOLVER.MAX_EPOCH + * cfg.MULTIGRID.EPOCH_FACTOR + / sum(s[-1] for s in schedule) + ) + + final_schedule = [] + total_epochs = 0 + for s in schedule: + epochs = s[2] * x + total_epochs += epochs + final_schedule.append((s[0], s[1], int(round(total_epochs)))) + print_schedule(final_schedule) + return final_schedule + + +def print_schedule(schedule): + """ + Log schedule. + """ + logger.info("Long cycle index\tBase shape\tEpochs") + for s in schedule: + logger.info("{}\t{}\t{}".format(s[0], s[1], s[2])) + + +def get_current_long_cycle_shape(schedule, epoch): + """ + Given a schedule and epoch index, return the long cycle base shape. + Args: + schedule (configs): configs that contains training and multigrid specific + hyperparameters. Details can be seen in + slowfast/config/defaults.py. + cur_epoch (int): current epoch index. + Returns: + shapes (list): A list describing the base shape in a long cycle: + [batch size relative to default, + number of frames, spatial dimension]. + """ + for s in schedule: + if epoch < s[-1]: + return s[1] + return schedule[-1][1] diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/multiprocessing.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/multiprocessing.py" new file mode 100644 index 0000000..09938f3 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/multiprocessing.py" @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Multiprocessing helpers.""" + +import torch + + +def run( + local_rank, + num_proc, + func, + init_method, + shard_id, + num_shards, + backend, + cfg, + output_queue=None, +): + """ + Runs a function from a child process. + Args: + local_rank (int): rank of the current process on the current machine. + num_proc (int): number of processes per machine. + func (function): function to execute on each of the process. + init_method (string): method to initialize the distributed training. + TCP initialization: equiring a network address reachable from all + processes followed by the port. + Shared file-system initialization: makes use of a file system that + is shared and visible from all machines. The URL should start with + file:// and contain a path to a non-existent file on a shared file + system. + shard_id (int): the rank of the current machine. + num_shards (int): number of overall machines for the distributed + training job. + backend (string): three distributed backends ('nccl', 'gloo', 'mpi') are + supports, each with different capabilities. Details can be found + here: + https://pytorch.org/docs/stable/distributed.html + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + output_queue (queue): can optionally be used to return values from the + master process. + """ + # Initialize the process group. + world_size = num_proc * num_shards + rank = shard_id * num_proc + local_rank + + try: + torch.distributed.init_process_group( + backend=backend, + init_method=init_method, + world_size=world_size, + rank=rank, + ) + except Exception as e: + raise e + + torch.cuda.set_device(local_rank) + ret = func(cfg) + if output_queue is not None and local_rank == 0: + output_queue.put(ret) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/parser.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/parser.py" new file mode 100644 index 0000000..c406511 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/parser.py" @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Argument parser functions.""" + +import argparse +import sys + +import slowfast.utils.checkpoint as cu +from slowfast.config.defaults import get_cfg + + +def parse_args(): + """ + Parse the following arguments for a default parser for PySlowFast users. + Args: + shard_id (int): shard id for the current machine. Starts from 0 to + num_shards - 1. If single machine is used, then set shard id to 0. + num_shards (int): number of shards using by the job. + init_method (str): initialization method to launch the job with multiple + devices. Options includes TCP or shared file-system for + initialization. details can be find in + https://pytorch.org/docs/stable/distributed.html#tcp-initialization + cfg (str): path to the config file. + opts (argument): provide addtional options from the command line, it + overwrites the config loaded from file. + """ + parser = argparse.ArgumentParser( + description="Provide SlowFast video training and testing pipeline." + ) + parser.add_argument( + "--shard_id", + help="The shard id of current node, Starts from 0 to num_shards - 1", + default=0, + type=int, + ) + parser.add_argument( + "--num_shards", + help="Number of shards using by the job", + default=1, + type=int, + ) + parser.add_argument( + "--init_method", + help="Initialization method, includes TCP or shared file-system", + default="tcp://localhost:9999", + type=str, + ) + parser.add_argument( + "--cfg", + dest="cfg_file", + help="Path to the config file", + default="configs/Kinetics/SLOWFAST_4x16_R50.yaml", + type=str, + ) + parser.add_argument( + "opts", + help="See slowfast/config/defaults.py for all options", + default=None, + nargs=argparse.REMAINDER, + ) + if len(sys.argv) == 1: + parser.print_help() + return parser.parse_args() + + +def load_config(args): + """ + Given the arguemnts, load and initialize the configs. + Args: + args (argument): arguments includes `shard_id`, `num_shards`, + `init_method`, `cfg_file`, and `opts`. + """ + # Setup cfg. + cfg = get_cfg() + # Load config from cfg. + if args.cfg_file is not None: + cfg.merge_from_file(args.cfg_file) + # Load config from command line, overwrite config from opts. + if args.opts is not None: + cfg.merge_from_list(args.opts) + + # Inherit parameters from args. + if hasattr(args, "num_shards") and hasattr(args, "shard_id"): + cfg.NUM_SHARDS = args.num_shards + cfg.SHARD_ID = args.shard_id + if hasattr(args, "rng_seed"): + cfg.RNG_SEED = args.rng_seed + if hasattr(args, "output_dir"): + cfg.OUTPUT_DIR = args.output_dir + + # Create the checkpoint dir. + cu.make_checkpoint_dir(cfg.OUTPUT_DIR) + return cfg diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/weight_init_helper.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/weight_init_helper.py" new file mode 100644 index 0000000..0a2d65d --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/utils/weight_init_helper.py" @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Utility function for weight initialization""" + +import torch.nn as nn +from fvcore.nn.weight_init import c2_msra_fill + + +def init_weights(model, fc_init_std=0.01, zero_init_final_bn=True): + """ + Performs ResNet style weight initialization. + Args: + fc_init_std (float): the expected standard deviation for fc layer. + zero_init_final_bn (bool): if True, zero initialize the final bn for + every bottleneck. + """ + for m in model.modules(): + if isinstance(m, nn.Conv3d): + """ + Follow the initialization method proposed in: + {He, Kaiming, et al. + "Delving deep into rectifiers: Surpassing human-level + performance on imagenet classification." + arXiv preprint arXiv:1502.01852 (2015)} + """ + c2_msra_fill(m) + elif isinstance(m, nn.BatchNorm3d): + if ( + hasattr(m, "transform_final_bn") + and m.transform_final_bn + and zero_init_final_bn + ): + batchnorm_weight = 0.0 + else: + batchnorm_weight = 1.0 + if m.weight is not None: + m.weight.data.fill_(batchnorm_weight) + if m.bias is not None: + m.bias.data.zero_() + if isinstance(m, nn.Linear): + m.weight.data.normal_(mean=0.0, std=fc_init_std) + if m.bias is not None: + m.bias.data.zero_() diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/__init__.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/__init__.py" new file mode 100644 index 0000000..8dbe96a --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/__init__.py" @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/async_predictor.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/async_predictor.py" new file mode 100644 index 0000000..bb11ef1 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/async_predictor.py" @@ -0,0 +1,319 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import atexit +import numpy as np +import queue +import torch +import torch.multiprocessing as mp + +import slowfast.utils.logging as logging +from slowfast.datasets import cv2_transform +from slowfast.visualization.predictor import Predictor + +logger = logging.get_logger(__name__) + + +class AsycnActionPredictor: + class _Predictor(mp.Process): + def __init__(self, cfg, task_queue, result_queue, gpu_id=None): + """ + Predict Worker for Detectron2. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + task_queue (mp.Queue): a shared queue for incoming task. + result_queue (mp.Queue): a shared queue for predicted results. + gpu_id (int): index of the GPU device for the current child process. + """ + super().__init__() + self.cfg = cfg + self.task_queue = task_queue + self.result_queue = result_queue + self.gpu_id = gpu_id + + self.device = ( + torch.device("cuda:{}".format(self.gpu_id)) + if self.cfg.NUM_GPUS + else "cpu" + ) + + def run(self): + """ + Run prediction asynchronously. + """ + # Build the video model and print model statistics. + model = Predictor(self.cfg, gpu_id=self.gpu_id) + while True: + task = self.task_queue.get() + if isinstance(task, _StopToken): + break + task = model(task) + self.result_queue.put(task) + + def __init__(self, cfg, result_queue=None): + num_workers = cfg.NUM_GPUS + + self.task_queue = mp.Queue() + self.result_queue = mp.Queue() if result_queue is None else result_queue + + self.get_idx = -1 + self.put_idx = -1 + self.procs = [] + cfg = cfg.clone() + cfg.defrost() + cfg.NUM_GPUS = 1 + for gpu_id in range(num_workers): + self.procs.append( + AsycnActionPredictor._Predictor( + cfg, self.task_queue, self.result_queue, gpu_id + ) + ) + + self.result_data = {} + for p in self.procs: + p.start() + atexit.register(self.shutdown) + + def put(self, task): + """ + Add the new task to task queue. + Args: + task (TaskInfo object): task object that contain + the necessary information for action prediction. (e.g. frames) + """ + self.put_idx += 1 + self.task_queue.put(task) + + def get(self): + """ + Return a task object in the correct order based on task id if + result(s) is available. Otherwise, raise queue.Empty exception. + """ + if self.result_data.get(self.get_idx + 1) is not None: + self.get_idx += 1 + res = self.result_data[self.get_idx] + del self.result_data[self.get_idx] + return res + while True: + res = self.result_queue.get(block=False) + idx = res.id + if idx == self.get_idx + 1: + self.get_idx += 1 + return res + self.result_data[idx] = res + + def __call__(self, task): + self.put(task) + return self.get() + + def shutdown(self): + for _ in self.procs: + self.task_queue.put(_StopToken()) + + @property + def result_available(self): + """ + How many results are ready to be returned. + """ + return self.result_queue.qsize() + len(self.result_data) + + @property + def default_buffer_size(self): + return len(self.procs) * 5 + + +class AsyncVis: + class _VisWorker(mp.Process): + def __init__(self, video_vis, task_queue, result_queue): + """ + Visualization Worker for AsyncVis. + Args: + video_vis (VideoVisualizer object): object with tools for visualization. + task_queue (mp.Queue): a shared queue for incoming task for visualization. + result_queue (mp.Queue): a shared queue for visualized results. + """ + self.video_vis = video_vis + self.task_queue = task_queue + self.result_queue = result_queue + super().__init__() + + def run(self): + """ + Run visualization asynchronously. + """ + while True: + task = self.task_queue.get() + if isinstance(task, _StopToken): + break + + frames = draw_predictions(task, self.video_vis) + task.frames = np.array(frames) + self.result_queue.put(task) + + def __init__(self, video_vis, n_workers=None): + """ + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + n_workers (Optional[int]): number of CPUs for running video visualizer. + If not given, use all CPUs. + """ + + num_workers = mp.cpu_count() if n_workers is None else n_workers + + self.task_queue = mp.Queue() + self.result_queue = mp.Queue() + self.get_indices_ls = [] + self.procs = [] + self.result_data = {} + self.put_id = -1 + for _ in range(max(num_workers, 1)): + self.procs.append( + AsyncVis._VisWorker( + video_vis, self.task_queue, self.result_queue + ) + ) + + for p in self.procs: + p.start() + + atexit.register(self.shutdown) + + def put(self, task): + """ + Add the new task to task queue. + Args: + task (TaskInfo object): task object that contain + the necessary information for action prediction. (e.g. frames, boxes, predictions) + """ + self.put_id += 1 + self.task_queue.put(task) + + def get(self): + """ + Return visualized frames/clips in the correct order based on task id if + result(s) is available. Otherwise, raise queue.Empty exception. + """ + get_idx = self.get_indices_ls[0] + if self.result_data.get(get_idx) is not None: + res = self.result_data[get_idx] + del self.result_data[get_idx] + del self.get_indices_ls[0] + return res + + while True: + res = self.result_queue.get(block=False) + idx = res.id + if idx == get_idx: + del self.get_indices_ls[0] + return res + self.result_data[idx] = res + + def __call__(self, task): + """ + How many results are ready to be returned. + """ + self.put(task) + return self.get() + + def shutdown(self): + for _ in self.procs: + self.task_queue.put(_StopToken()) + + @property + def result_available(self): + return self.result_queue.qsize() + len(self.result_data) + + @property + def default_buffer_size(self): + return len(self.procs) * 5 + + +class _StopToken: + pass + + +class AsyncDemo: + """ + Asynchronous Action Prediction and Visualization pipeline with AsyncVis. + """ + + def __init__(self, cfg, async_vis): + """ + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + async_vis (AsyncVis object): asynchronous visualizer. + """ + self.model = AsycnActionPredictor( + cfg=cfg, result_queue=async_vis.task_queue + ) + self.async_vis = async_vis + + def put(self, task): + """ + Put task into task queue for prediction and visualization. + Args: + task (TaskInfo object): task object that contain + the necessary information for action prediction. (e.g. frames) + """ + self.async_vis.get_indices_ls.append(task.id) + self.model.put(task) + + def get(self): + """ + Get the visualized clips if any. + """ + try: + task = self.async_vis.get() + except (queue.Empty, IndexError): + raise IndexError("Results are not available yet.") + + return task + + +def draw_predictions(task, video_vis): + """ + Draw prediction for the given task. + Args: + task (TaskInfo object): task object that contain + the necessary information for visualization. (e.g. frames, preds) + All attributes must lie on CPU devices. + video_vis (VideoVisualizer object): the video visualizer object. + """ + boxes = task.bboxes + frames = task.frames + preds = task.action_preds + if boxes is not None: + img_width = task.img_width + img_height = task.img_height + if boxes.device != torch.device("cpu"): + boxes = boxes.cpu() + boxes = cv2_transform.revert_scaled_boxes( + task.crop_size, boxes, img_height, img_width + ) + + keyframe_idx = len(frames) // 2 - task.num_buffer_frames + draw_range = [ + keyframe_idx - task.clip_vis_size, + keyframe_idx + task.clip_vis_size, + ] + buffer = frames[: task.num_buffer_frames] + frames = frames[task.num_buffer_frames :] + if boxes is not None: + if len(boxes) != 0: + frames = video_vis.draw_clip_range( + frames, + preds, + boxes, + keyframe_idx=keyframe_idx, + draw_range=draw_range, + ) + else: + frames = video_vis.draw_clip_range( + frames, preds, keyframe_idx=keyframe_idx, draw_range=draw_range + ) + del task + + return buffer + frames diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/ava_demo_precomputed_boxes.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/ava_demo_precomputed_boxes.py" new file mode 100644 index 0000000..1c97ad4 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/ava_demo_precomputed_boxes.py" @@ -0,0 +1,467 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import numpy as np +import os +import cv2 +import torch +import tqdm +from iopath.common.file_io import g_pathmgr + +import slowfast.utils.checkpoint as cu +import slowfast.utils.logging as logging +from slowfast.datasets.ava_helper import parse_bboxes_file +from slowfast.datasets.cv2_transform import scale, scale_boxes +from slowfast.datasets.utils import get_sequence +from slowfast.models import build_model +from slowfast.utils import misc +from slowfast.visualization.utils import process_cv2_inputs +from slowfast.visualization.video_visualizer import VideoVisualizer + +logger = logging.get_logger(__name__) + + +class AVAVisualizerWithPrecomputedBox: + """ + Visualize action predictions for videos or folder of images with precomputed + and ground-truth boxes in AVA format. + """ + + def __init__(self, cfg): + """ + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + self.source = g_pathmgr.get_local_path(path=cfg.DEMO.INPUT_VIDEO) + self.fps = None + if g_pathmgr.isdir(self.source): + self.fps = cfg.DEMO.FPS + self.video_name = self.source.split("/")[-1] + self.source = os.path.join( + self.source, "{}_%06d.jpg".format(self.video_name) + ) + else: + self.video_name = self.source.split("/")[-1] + self.video_name = self.video_name.split(".")[0] + + self.cfg = cfg + self.cap = cv2.VideoCapture(self.source) + if self.fps is None: + self.fps = self.cap.get(cv2.CAP_PROP_FPS) + + self.total_frames = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT)) + + self.display_width = int(self.cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + self.display_height = int(self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + + if not self.cap.isOpened(): + raise IOError("Video {} cannot be opened".format(self.source)) + + self.output_file = None + + if cfg.DEMO.OUTPUT_FILE != "": + self.output_file = self.get_output_file(cfg.DEMO.OUTPUT_FILE) + + self.pred_boxes, self.gt_boxes = load_boxes_labels( + cfg, + self.video_name, + self.fps, + self.display_width, + self.display_height, + ) + + self.seq_length = cfg.DATA.NUM_FRAMES * cfg.DATA.SAMPLING_RATE + self.no_frames_repeat = cfg.DEMO.SLOWMO + + def get_output_file(self, path): + """ + Return a video writer object. + Args: + path (str): path to the output video file. + """ + return cv2.VideoWriter( + filename=path, + fourcc=cv2.VideoWriter_fourcc(*"mp4v"), + fps=float(30), + frameSize=(self.display_width, self.display_height), + isColor=True, + ) + + def get_input_clip(self, keyframe_idx): + """ + Get input clip from the video/folder of images for a given + keyframe index. + Args: + keyframe_idx (int): index of the current keyframe. + Returns: + clip (list of tensors): formatted input clip(s) corresponding to + the current keyframe. + """ + seq = get_sequence( + keyframe_idx, + self.seq_length // 2, + self.cfg.DATA.SAMPLING_RATE, + self.total_frames, + ) + clip = [] + for frame_idx in seq: + self.cap.set(cv2.CAP_PROP_POS_FRAMES, frame_idx) + was_read, frame = self.cap.read() + if was_read: + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + frame = scale(self.cfg.DATA.TEST_CROP_SIZE, frame) + clip.append(frame) + else: + logger.error( + "Unable to read frame. Duplicating previous frame." + ) + clip.append(clip[-1]) + + clip = process_cv2_inputs(clip, self.cfg) + return clip + + def get_predictions(self): + """ + Predict and append prediction results to each box in each keyframe in + `self.pred_boxes` dictionary. + """ + # Set random seed from configs. + np.random.seed(self.cfg.RNG_SEED) + torch.manual_seed(self.cfg.RNG_SEED) + + # Setup logging format. + logging.setup_logging(self.cfg.OUTPUT_DIR) + + # Print config. + logger.info("Run demo with config:") + logger.info(self.cfg) + assert ( + self.cfg.NUM_GPUS <= 1 + ), "Cannot run demo visualization on multiple GPUs." + + # Build the video model and print model statistics. + model = build_model(self.cfg) + model.eval() + logger.info("Start loading model info") + misc.log_model_info(model, self.cfg, use_train_input=False) + logger.info("Start loading model weights") + cu.load_test_checkpoint(self.cfg, model) + logger.info("Finish loading model weights") + logger.info("Start making predictions for precomputed boxes.") + for keyframe_idx, boxes_and_labels in tqdm.tqdm( + self.pred_boxes.items() + ): + inputs = self.get_input_clip(keyframe_idx) + boxes = boxes_and_labels[0] + boxes = torch.from_numpy(np.array(boxes)).float() + + box_transformed = scale_boxes( + self.cfg.DATA.TEST_CROP_SIZE, + boxes, + self.display_height, + self.display_width, + ) + + # Pad frame index for each box. + box_inputs = torch.cat( + [ + torch.full((box_transformed.shape[0], 1), float(0)), + box_transformed, + ], + axis=1, + ) + if self.cfg.NUM_GPUS: + # Transfer the data to the current GPU device. + if isinstance(inputs, (list,)): + for i in range(len(inputs)): + inputs[i] = inputs[i].cuda(non_blocking=True) + else: + inputs = inputs.cuda(non_blocking=True) + + box_inputs = box_inputs.cuda() + + preds = model(inputs, box_inputs) + + preds = preds.detach() + + if self.cfg.NUM_GPUS: + preds = preds.cpu() + + boxes_and_labels[1] = preds + + def draw_video(self): + """ + Draw predicted and ground-truth (if provided) results on the video/folder of images. + Write the visualized result to a video output file. + """ + all_boxes = merge_pred_gt_boxes(self.pred_boxes, self.gt_boxes) + common_classes = ( + self.cfg.DEMO.COMMON_CLASS_NAMES + if len(self.cfg.DEMO.LABEL_FILE_PATH) != 0 + else None + ) + video_vis = VideoVisualizer( + num_classes=self.cfg.MODEL.NUM_CLASSES, + class_names_path=self.cfg.DEMO.LABEL_FILE_PATH, + top_k=self.cfg.TENSORBOARD.MODEL_VIS.TOPK_PREDS, + thres=self.cfg.DEMO.COMMON_CLASS_THRES, + lower_thres=self.cfg.DEMO.UNCOMMON_CLASS_THRES, + common_class_names=common_classes, + colormap=self.cfg.TENSORBOARD.MODEL_VIS.COLORMAP, + mode=self.cfg.DEMO.VIS_MODE, + ) + + all_keys = sorted(all_boxes.keys()) + # Draw around the keyframe for 2/10 of the sequence length. + # This is chosen using heuristics. + draw_range = [ + self.seq_length // 2 - self.seq_length // 10, + self.seq_length // 2 + self.seq_length // 10, + ] + draw_range_repeat = [ + draw_range[0], + (draw_range[1] - draw_range[0]) * self.no_frames_repeat + + draw_range[0], + ] + prev_buffer = [] + prev_end_idx = 0 + + logger.info("Start Visualization...") + for keyframe_idx in tqdm.tqdm(all_keys): + pred_gt_boxes = all_boxes[keyframe_idx] + # Find the starting index of the clip. If start_idx exceeds the beginning + # of the video, we only choose valid frame from index 0. + start_idx = max(0, keyframe_idx - self.seq_length // 2) + # Number of frames from the start of the current clip and the + # end of the previous clip. + dist = start_idx - prev_end_idx + # If there are unwritten frames in between clips. + if dist >= 0: + # Get the frames in between previous clip and current clip. + frames = self._get_frame_range(prev_end_idx, dist) + # We keep a buffer of frames for overlapping visualization. + # Write these to the output file. + for frame in prev_buffer: + frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) + self.display(frame) + # Write them to output file without any visualization + # since they don't have any corresponding keyframes. + for frame in frames: + self.display(frame) + prev_buffer = [] + num_new_frames = self.seq_length + + # If there are overlapping frames in between clips. + elif dist < 0: + # Flush all ready frames. + for frame in prev_buffer[:dist]: + frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) + self.display(frame) + prev_buffer = prev_buffer[dist:] + num_new_frames = self.seq_length + dist + # Obtain new frames for the current clip from the input video file. + new_frames = self._get_frame_range( + max(start_idx, prev_end_idx), num_new_frames + ) + new_frames = [ + cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) for frame in new_frames + ] + clip = prev_buffer + new_frames + # Calculate the end of this clip. This will be `prev_end_idx` for the + # next iteration. + prev_end_idx = max(start_idx, prev_end_idx) + len(new_frames) + # For each precomputed or gt boxes. + for i, boxes in enumerate(pred_gt_boxes): + if i == 0: + repeat = self.no_frames_repeat + current_draw_range = draw_range + else: + repeat = 1 + current_draw_range = draw_range_repeat + # Make sure draw range does not fall out of end of clip. + current_draw_range[1] = min( + current_draw_range[1], len(clip) - 1 + ) + ground_truth = boxes[0] + bboxes = boxes[1] + label = boxes[2] + # Draw predictions. + clip = video_vis.draw_clip_range( + clip, + label, + bboxes=torch.Tensor(bboxes), + ground_truth=ground_truth, + draw_range=current_draw_range, + repeat_frame=repeat, + ) + # Store the current clip as buffer. + prev_buffer = clip + + # Write the remaining buffer to output file. + for frame in prev_buffer: + frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) + self.display(frame) + # If we still have some remaining frames in the input file, + # write those to the output file as well. + if prev_end_idx < self.total_frames: + dist = self.total_frames - prev_end_idx + remaining_clip = self._get_frame_range(prev_end_idx, dist) + for frame in remaining_clip: + self.display(frame) + + def __call__(self): + self.get_predictions() + self.draw_video() + + def display(self, frame): + """ + Either display a single frame (BGR image) to a window or write to + an output file if output path is provided. + """ + if self.output_file is None: + cv2.imshow("SlowFast", frame) + else: + self.output_file.write(frame) + + def _get_keyframe_clip(self, keyframe_idx): + """ + Return a clip corresponding to a keyframe index for visualization. + Args: + keyframe_idx (int): keyframe index. + """ + start_idx = max(0, keyframe_idx - self.seq_length // 2) + + clip = self._get_frame_range(start_idx, self.seq_length) + + return clip + + def _get_frame_range(self, start_idx, num_frames): + """ + Return a clip of `num_frames` frames starting from `start_idx`. If not enough frames + from `start_idx`, return the remaining frames from `start_idx`. + Args: + start_idx (int): starting idx. + num_frames (int): number of frames in the returned clip. + """ + was_read = True + assert start_idx < self.total_frames, "Start index out of range." + + self.cap.set(cv2.CAP_PROP_POS_FRAMES, start_idx) + all_frames = [] + for _ in range(num_frames): + was_read, frame = self.cap.read() + if was_read: + all_frames.append(frame) + else: + break + + return all_frames + + +def merge_pred_gt_boxes(pred_dict, gt_dict=None): + """ + Merge data from precomputed and ground-truth boxes dictionaries. + Args: + pred_dict (dict): a dict which maps from `frame_idx` to a list of `boxes` + and `labels`. Each `box` is a list of 4 box coordinates. `labels[i]` is + a list of labels for `boxes[i]`. + gt_dict (Optional[dict]): a dict which maps from `frame_idx` to a list of `boxes` + and `labels`. Each `box` is a list of 4 box coordinates. `labels[i]` is + a list of labels for `boxes[i]`. Note that label is -1 for predicted boxes. + Returns: + merged_dict (dict): merged dictionary from `pred_dict` and `gt_dict` if given. + It is a dict which maps from `frame_idx` to a list of [`is_gt`, `boxes`, `labels`], + where `is_gt` is a boolean indicate whether the `boxes` and `labels` are ground-truth. + """ + merged_dict = {} + for key, item in pred_dict.items(): + merged_dict[key] = [[False, item[0], item[1]]] + + if gt_dict is not None: + for key, item in gt_dict.items(): + if merged_dict.get(key) is None: + merged_dict[key] = [[True, item[0], item[1]]] + else: + merged_dict[key].append([True, item[0], item[1]]) + return merged_dict + + +def load_boxes_labels(cfg, video_name, fps, img_width, img_height): + """ + Loading boxes and labels from AVA bounding boxes csv files. + Args: + cfg (CfgNode): config. + video_name (str): name of the given video. + fps (int or float): frames per second of the input video/images folder. + img_width (int): width of images in input video/images folder. + img_height (int): height of images in input video/images folder. + Returns: + preds_boxes (dict): a dict which maps from `frame_idx` to a list of `boxes` + and `labels`. Each `box` is a list of 4 box coordinates. `labels[i]` is + a list of labels for `boxes[i]`. Note that label is -1 for predicted boxes. + gt_boxes (dict): if cfg.DEMO.GT_BOXES is given, return similar dict as + all_pred_boxes but for ground-truth boxes. + """ + starting_second = cfg.DEMO.STARTING_SECOND + + def sec_to_frameidx(sec): + return (sec - starting_second) * fps + + def process_bboxes_dict(dictionary): + """ + Replace all `keyframe_sec` in `dictionary` with `keyframe_idx` and + merge all [`box_coordinate`, `box_labels`] pairs into + [`all_boxes_coordinates`, `all_boxes_labels`] for each `keyframe_idx`. + Args: + dictionary (dict): a dictionary which maps `frame_sec` to a list of `box`. + Each `box` is a [`box_coord`, `box_labels`] where `box_coord` is the + coordinates of box and 'box_labels` are the corresponding + labels for the box. + Returns: + new_dict (dict): a dict which maps from `frame_idx` to a list of `boxes` + and `labels`. Each `box` in `boxes` is a list of 4 box coordinates. `labels[i]` + is a list of labels for `boxes[i]`. Note that label is -1 for predicted boxes. + """ + # Replace all keyframe_sec with keyframe_idx. + new_dict = {} + for keyframe_sec, boxes_and_labels in dictionary.items(): + # Ignore keyframes with no boxes + if len(boxes_and_labels) == 0: + continue + keyframe_idx = sec_to_frameidx(keyframe_sec) + boxes, labels = list(zip(*boxes_and_labels)) + # Shift labels from [1, n_classes] to [0, n_classes - 1]. + labels = [[i - 1 for i in box_label] for box_label in labels] + boxes = np.array(boxes) + boxes[:, [0, 2]] *= img_width + boxes[:, [1, 3]] *= img_height + new_dict[keyframe_idx] = [boxes.tolist(), list(labels)] + return new_dict + + preds_boxes_path = cfg.DEMO.PREDS_BOXES + gt_boxes_path = cfg.DEMO.GT_BOXES + + preds_boxes, _, _ = parse_bboxes_file( + ann_filenames=[preds_boxes_path], + ann_is_gt_box=[False], + detect_thresh=cfg.AVA.DETECTION_SCORE_THRESH, + boxes_sample_rate=1, + ) + preds_boxes = preds_boxes[video_name] + if gt_boxes_path == "": + gt_boxes = None + else: + gt_boxes, _, _ = parse_bboxes_file( + ann_filenames=[gt_boxes_path], + ann_is_gt_box=[True], + detect_thresh=cfg.AVA.DETECTION_SCORE_THRESH, + boxes_sample_rate=1, + ) + gt_boxes = gt_boxes[video_name] + + preds_boxes = process_bboxes_dict(preds_boxes) + if gt_boxes is not None: + gt_boxes = process_bboxes_dict(gt_boxes) + + return preds_boxes, gt_boxes diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/demo_loader.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/demo_loader.py" new file mode 100644 index 0000000..13443d6 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/demo_loader.py" @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import atexit +import copy +import queue +import threading +import time +import cv2 + +import slowfast.utils.logging as logging +from slowfast.visualization.utils import TaskInfo + +logger = logging.get_logger(__name__) + + +class VideoManager: + """ + VideoManager object for getting frames from video source for inference. + """ + + def __init__(self, cfg): + """ + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + assert ( + cfg.DEMO.WEBCAM > -1 or cfg.DEMO.INPUT_VIDEO != "" + ), "Must specify a data source as input." + + self.source = ( + cfg.DEMO.WEBCAM if cfg.DEMO.WEBCAM > -1 else cfg.DEMO.INPUT_VIDEO + ) + + self.display_width = cfg.DEMO.DISPLAY_WIDTH + self.display_height = cfg.DEMO.DISPLAY_HEIGHT + + self.cap = cv2.VideoCapture(self.source) + + if self.display_width > 0 and self.display_height > 0: + self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, self.display_width) + self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self.display_height) + else: + self.display_width = int(self.cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + self.display_height = int(self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + + if not self.cap.isOpened(): + raise IOError("Video {} cannot be opened".format(self.source)) + + self.output_file = None + if cfg.DEMO.OUTPUT_FPS == -1: + self.output_fps = self.cap.get(cv2.CAP_PROP_FPS) + else: + self.output_fps = cfg.DEMO.OUTPUT_FPS + if cfg.DEMO.OUTPUT_FILE != "": + self.output_file = self.get_output_file( + cfg.DEMO.OUTPUT_FILE, fps=self.output_fps + ) + self.id = -1 + self.buffer = [] + self.buffer_size = cfg.DEMO.BUFFER_SIZE + self.seq_length = cfg.DATA.NUM_FRAMES * cfg.DATA.SAMPLING_RATE + self.test_crop_size = cfg.DATA.TEST_CROP_SIZE + self.clip_vis_size = cfg.DEMO.CLIP_VIS_SIZE + + def __iter__(self): + return self + + def __next__(self): + """ + Read and return the required number of frames for 1 clip. + Returns: + was_read (bool): False if not enough frames to return. + task (TaskInfo object): object contains metadata for the current clips. + """ + self.id += 1 + task = TaskInfo() + + task.img_height = self.display_height + task.img_width = self.display_width + task.crop_size = self.test_crop_size + task.clip_vis_size = self.clip_vis_size + + frames = [] + if len(self.buffer) != 0: + frames = self.buffer + was_read = True + while was_read and len(frames) < self.seq_length: + was_read, frame = self.cap.read() + print(frame) + frames.append(frame) + if was_read and self.buffer_size != 0: + self.buffer = frames[-self.buffer_size :] + + task.add_frames(self.id, frames) + task.num_buffer_frames = 0 if self.id == 0 else self.buffer_size + + return was_read, task + + def get_output_file(self, path, fps=30): + """ + Return a video writer object. + Args: + path (str): path to the output video file. + fps (int or float): frames per second. + """ + return cv2.VideoWriter( + filename=path, + fourcc=cv2.VideoWriter_fourcc(*"mp4v"), + fps=float(fps), + frameSize=(self.display_width, self.display_height), + isColor=True, + ) + + def display(self, task): + """ + Either display a single frame (BGR image) to a window or write to + an output file if output path is provided. + Args: + task (TaskInfo object): task object that contain + the necessary information for prediction visualization. (e.g. visualized frames.) + """ + for frame in task.frames[task.num_buffer_frames :]: + if self.output_file is None: + cv2.imshow("SlowFast", frame) + time.sleep(1 / self.output_fps) + else: + self.output_file.write(frame) + + def clean(self): + """ + Clean up open video files and windows. + """ + self.cap.release() + if self.output_file is None: + cv2.destroyAllWindows() + else: + self.output_file.release() + + def start(self): + return self + + def join(self): + pass + + +class ThreadVideoManager: + """ + VideoManager object for getting frames from video source for inference + using multithreading for read and write frames. + """ + + def __init__(self, cfg): + """ + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + assert ( + cfg.DEMO.WEBCAM > -1 or cfg.DEMO.INPUT_VIDEO != "" + ), "Must specify a data source as input." + + self.source = ( + cfg.DEMO.WEBCAM if cfg.DEMO.WEBCAM > -1 else cfg.DEMO.INPUT_VIDEO + ) + + self.display_width = cfg.DEMO.DISPLAY_WIDTH + self.display_height = cfg.DEMO.DISPLAY_HEIGHT + + self.cap = cv2.VideoCapture(self.source) + + if self.display_width > 0 and self.display_height > 0: + self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, self.display_width) + self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self.display_height) + else: + self.display_width = int(self.cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + self.display_height = int(self.cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + + if not self.cap.isOpened(): + raise IOError("Video {} cannot be opened".format(self.source)) + + self.output_file = None + + if cfg.DEMO.OUTPUT_FPS == -1: + self.output_fps = self.cap.get(cv2.CAP_PROP_FPS) + else: + self.output_fps = cfg.DEMO.OUTPUT_FPS + if cfg.DEMO.OUTPUT_FILE != "": + self.output_file = self.get_output_file( + cfg.DEMO.OUTPUT_FILE, fps=self.output_fps + ) + self.num_skip = cfg.DEMO.NUM_CLIPS_SKIP + 1 + self.get_id = -1 + self.put_id = -1 + self.buffer = [] + self.buffer_size = cfg.DEMO.BUFFER_SIZE + self.seq_length = cfg.DATA.NUM_FRAMES * cfg.DATA.SAMPLING_RATE + self.test_crop_size = cfg.DATA.TEST_CROP_SIZE + self.clip_vis_size = cfg.DEMO.CLIP_VIS_SIZE + + self.read_queue = queue.Queue() + self.write_queue = {} + self.not_end = True + self.write_lock = threading.Lock() + self.put_id_lock = threading.Lock() + self.input_lock = threading.Lock() + self.output_lock = threading.Lock() + self.stopped = False + atexit.register(self.clean) + + def get_output_file(self, path, fps=30): + """ + Return a video writer object. + Args: + path (str): path to the output video file. + fps (int or float): frames per second. + """ + return cv2.VideoWriter( + filename=path, + fourcc=cv2.VideoWriter_fourcc(*"mp4v"), + fps=float(fps), + frameSize=(self.display_width, self.display_height), + isColor=True, + ) + + def __iter__(self): + return self + + def put_fn(self): + """ + Grabbing frames from VideoCapture. + """ + was_read = True + while was_read and not self.stopped: + task = TaskInfo() + + task.img_height = self.display_height + task.img_width = self.display_width + task.crop_size = self.test_crop_size + task.clip_vis_size = self.clip_vis_size + frames = [] + if len(self.buffer) != 0: + frames = self.buffer + self.input_lock.acquire() + while was_read and len(frames) < self.seq_length: + was_read, frame = self.cap.read() + if was_read: + frames.append(frame) + self.input_lock.release() + if was_read: + self.buffer = frames[-self.buffer_size :] + + task.add_frames(self.put_id + 1, frames) + task.num_buffer_frames = ( + 0 if self.put_id == -1 else self.buffer_size + ) + with self.put_id_lock: + self.put_id += 1 + self.not_end = was_read + # If mode is to read the most recent clip or we reach task + # index that is not supposed to be skipped. + if self.num_skip == 0 or self.put_id % self.num_skip == 0: + self.read_queue.put((was_read, copy.deepcopy(task))) + else: + with self.write_lock: + self.write_queue[task.id] = (was_read, copy.deepcopy(task)) + + def __next__(self): + # If there is nothing in the task queue. + if self.read_queue.qsize() == 0: + return self.not_end, None + else: + with self.put_id_lock: + put_id = self.put_id + was_read, task = None, None + # If mode is to predict most recent read clip. + if self.num_skip == 0: + # Write all previous clips to write queue. + with self.write_lock: + while True: + was_read, task = self.read_queue.get() + if task.id == put_id: + break + self.write_queue[task.id] = (was_read, task) + else: + was_read, task = self.read_queue.get() + # If we reach the end of the video. + if not was_read: + # Put to write queue. + with self.write_lock: + self.write_queue[put_id] = was_read, copy.deepcopy(task) + task = None + return was_read, task + + def get_fn(self): + while not self.stopped: + with self.put_id_lock: + put_id = self.put_id + not_end = self.not_end + + with self.write_lock: + # If video ended and we have display all frames. + if not not_end and self.get_id == put_id: + break + # If the next frames are not available, wait. + if ( + len(self.write_queue) == 0 + or self.write_queue.get(self.get_id + 1) is None + ): + time.sleep(0.02) + continue + else: + self.get_id += 1 + was_read, task = self.write_queue[self.get_id] + del self.write_queue[self.get_id] + + with self.output_lock: + for frame in task.frames[task.num_buffer_frames :]: + if self.output_file is None: + cv2.imshow("SlowFast", frame) + time.sleep(1 / self.output_fps) + else: + self.output_file.write(frame) + + def display(self, task): + """ + Add the visualized task to the write queue for display/write to outputfile. + Args: + task (TaskInfo object): task object that contain + the necessary information for prediction visualization. (e.g. visualized frames.) + """ + with self.write_lock: + self.write_queue[task.id] = (True, task) + + def start(self): + """ + Start threads to read and write frames. + """ + self.put_thread = threading.Thread( + target=self.put_fn, args=(), name="VidRead-Thread", daemon=True + ) + self.put_thread.start() + self.get_thread = threading.Thread( + target=self.get_fn, args=(), name="VidDisplay-Thread", daemon=True + ) + self.get_thread.start() + + return self + + def join(self): + self.get_thread.join() + + def clean(self): + """ + Clean up open video files and windows. + """ + self.stopped = True + self.input_lock.acquire() + self.cap.release() + self.input_lock.release() + self.output_lock.acquire() + if self.output_file is None: + cv2.destroyAllWindows() + else: + self.output_file.release() + self.output_lock.release() diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/gradcam_utils.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/gradcam_utils.py" new file mode 100644 index 0000000..07c0fa2 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/gradcam_utils.py" @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import matplotlib.pyplot as plt +import torch +import torch.nn.functional as F + +import slowfast.datasets.utils as data_utils +from slowfast.visualization.utils import get_layer + + +class GradCAM: + """ + GradCAM class helps create localization maps using the Grad-CAM method for input videos + and overlap the maps over the input videos as heatmaps. + https://arxiv.org/pdf/1610.02391.pdf + """ + + def __init__( + self, model, target_layers, data_mean, data_std, colormap="viridis" + ): + """ + Args: + model (model): the model to be used. + target_layers (list of str(s)): name of convolutional layer to be used to get + gradients and feature maps from for creating localization maps. + data_mean (tensor or list): mean value to add to input videos. + data_std (tensor or list): std to multiply for input videos. + colormap (Optional[str]): matplotlib colormap used to create heatmap. + See https://matplotlib.org/3.3.0/tutorials/colors/colormaps.html + """ + + self.model = model + # Run in eval mode. + self.model.eval() + self.target_layers = target_layers + + self.gradients = {} + self.activations = {} + self.colormap = plt.get_cmap(colormap) + self.data_mean = data_mean + self.data_std = data_std + self._register_hooks() + + def _register_single_hook(self, layer_name): + """ + Register forward and backward hook to a layer, given layer_name, + to obtain gradients and activations. + Args: + layer_name (str): name of the layer. + """ + + def get_gradients(module, grad_input, grad_output): + self.gradients[layer_name] = grad_output[0].detach() + + def get_activations(module, input, output): + self.activations[layer_name] = output.clone().detach() + + target_layer = get_layer(self.model, layer_name=layer_name) + target_layer.register_forward_hook(get_activations) + target_layer.register_backward_hook(get_gradients) + + def _register_hooks(self): + """ + Register hooks to layers in `self.target_layers`. + """ + for layer_name in self.target_layers: + self._register_single_hook(layer_name=layer_name) + + def _calculate_localization_map(self, inputs, labels=None): + """ + Calculate localization map for all inputs with Grad-CAM. + Args: + inputs (list of tensor(s)): the input clips. + labels (Optional[tensor]): labels of the current input clips. + Returns: + localization_maps (list of ndarray(s)): the localization map for + each corresponding input. + preds (tensor): shape (n_instances, n_class). Model predictions for `inputs`. + """ + assert len(inputs) == len( + self.target_layers + ), "Must register the same number of target layers as the number of input pathways." + input_clone = [inp.clone() for inp in inputs] + preds = self.model(input_clone) + + if labels is None: + score = torch.max(preds, dim=-1)[0] + else: + if labels.ndim == 1: + labels = labels.unsqueeze(-1) + score = torch.gather(preds, dim=1, index=labels) + + self.model.zero_grad() + score = torch.sum(score) + score.backward() + localization_maps = [] + for i, inp in enumerate(inputs): + _, _, T, H, W = inp.size() + + gradients = self.gradients[self.target_layers[i]] + activations = self.activations[self.target_layers[i]] + B, C, Tg, _, _ = gradients.size() + + weights = torch.mean(gradients.view(B, C, Tg, -1), dim=3) + + weights = weights.view(B, C, Tg, 1, 1) + localization_map = torch.sum( + weights * activations, dim=1, keepdim=True + ) + localization_map = F.relu(localization_map) + localization_map = F.interpolate( + localization_map, + size=(T, H, W), + mode="trilinear", + align_corners=False, + ) + localization_map_min, localization_map_max = ( + torch.min(localization_map.view(B, -1), dim=-1, keepdim=True)[ + 0 + ], + torch.max(localization_map.view(B, -1), dim=-1, keepdim=True)[ + 0 + ], + ) + localization_map_min = torch.reshape( + localization_map_min, shape=(B, 1, 1, 1, 1) + ) + localization_map_max = torch.reshape( + localization_map_max, shape=(B, 1, 1, 1, 1) + ) + # Normalize the localization map. + localization_map = (localization_map - localization_map_min) / ( + localization_map_max - localization_map_min + 1e-6 + ) + localization_map = localization_map.data + + localization_maps.append(localization_map) + + return localization_maps, preds + + def __call__(self, inputs, labels=None, alpha=0.5): + """ + Visualize the localization maps on their corresponding inputs as heatmap, + using Grad-CAM. + Args: + inputs (list of tensor(s)): the input clips. + labels (Optional[tensor]): labels of the current input clips. + alpha (float): transparency level of the heatmap, in the range [0, 1]. + Returns: + result_ls (list of tensor(s)): the visualized inputs. + preds (tensor): shape (n_instances, n_class). Model predictions for `inputs`. + """ + result_ls = [] + localization_maps, preds = self._calculate_localization_map( + inputs, labels=labels + ) + for i, localization_map in enumerate(localization_maps): + # Convert (B, 1, T, H, W) to (B, T, H, W) + localization_map = localization_map.squeeze(dim=1) + if localization_map.device != torch.device("cpu"): + localization_map = localization_map.cpu() + heatmap = self.colormap(localization_map) + heatmap = heatmap[:, :, :, :, :3] + # Permute input from (B, C, T, H, W) to (B, T, H, W, C) + curr_inp = inputs[i].permute(0, 2, 3, 4, 1) + if curr_inp.device != torch.device("cpu"): + curr_inp = curr_inp.cpu() + curr_inp = data_utils.revert_tensor_normalize( + curr_inp, self.data_mean, self.data_std + ) + heatmap = torch.from_numpy(heatmap) + curr_inp = alpha * heatmap + (1 - alpha) * curr_inp + # Permute inp to (B, T, C, H, W) + curr_inp = curr_inp.permute(0, 1, 4, 2, 3) + result_ls.append(curr_inp) + + return result_ls, preds diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/prediction_vis.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/prediction_vis.py" new file mode 100644 index 0000000..e2189b7 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/prediction_vis.py" @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import numpy as np +import torch + +import slowfast.datasets.utils as data_utils +import slowfast.utils.logging as logging +import slowfast.visualization.tensorboard_vis as tb +from slowfast.utils.misc import get_class_names +from slowfast.visualization.video_visualizer import VideoVisualizer + +logger = logging.get_logger(__name__) + + +class WrongPredictionVis: + """ + WrongPredictionVis class for visualizing video inputs to Tensorboard + for instances that the model makes wrong predictions. + """ + + def __init__(self, cfg): + """ + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + self.cfg = cfg + self.class_names, _, self.subset = get_class_names( + cfg.TENSORBOARD.CLASS_NAMES_PATH, + subset_path=cfg.TENSORBOARD.WRONG_PRED_VIS.SUBSET_PATH, + ) + if self.subset is not None: + self.subset = set(self.subset) + self.num_class = cfg.MODEL.NUM_CLASSES + self.video_vis = VideoVisualizer( + cfg.MODEL.NUM_CLASSES, + cfg.TENSORBOARD.CLASS_NAMES_PATH, + 1, + cfg.TENSORBOARD.MODEL_VIS.COLORMAP, + ) + self.tag = cfg.TENSORBOARD.WRONG_PRED_VIS.TAG + self.writer = tb.TensorboardWriter(cfg) + self.model_incorrect_classes = set() + + def _pick_wrong_preds(self, labels, preds): + """ + Returns a 1D tensor that contains the indices of instances that have + wrong predictions, where true labels in in the specified subset. + Args: + labels (tensor): tensor of shape (n_instances,) containing class ids. + preds (tensor): class scores from model, shape (n_intances, n_classes) + Returns: + mask (tensor): boolean tensor. `mask[i]` is True if `model` makes a wrong prediction. + """ + subset_mask = torch.ones(size=(len(labels),), dtype=torch.bool) + if self.subset is not None: + for i, label in enumerate(labels): + if label not in self.subset: + subset_mask[i] = False + + preds_ids = torch.argmax(preds, dim=-1) + + mask = preds_ids != labels + mask &= subset_mask + for i, wrong_pred in enumerate(mask): + if wrong_pred: + self.model_incorrect_classes.add(labels[i]) + + return mask + + def visualize_vid(self, video_input, labels, preds, batch_idx): + """ + Draw predicted labels on video inputs and visualize all incorrectly classified + videos in the current batch. + Args: + video_input (list of list of tensor(s)): list of videos for all pathways. + labels (array-like): shape (n_instances,) of true label for each instance. + preds (tensor): shape (n, instances, n_classes). The predicted scores for all instances. + tag (Optional[str]): all visualized video will be added under this tag. This is for organization + purposes in Tensorboard. + batch_idx (int): batch index of the current videos. + """ + + def add_video(vid, preds, tag, true_class_name): + """ + Draw predicted label on video and add it to Tensorboard. + Args: + vid (array-like): shape (C, T, H, W). Each image in `vid` is a RGB image. + preds (tensor): shape (n_classes,) or (1, n_classes). The predicted scores + for the current `vid`. + tag (str): tag for `vid` in Tensorboard. + true_class_name (str): the ground-truth class name of the current `vid` instance. + """ + # Permute to (T, H, W, C). + vid = vid.permute(1, 2, 3, 0) + vid = data_utils.revert_tensor_normalize( + vid.cpu(), self.cfg.DATA.MEAN, self.cfg.DATA.STD + ) + vid = self.video_vis.draw_clip(vid, preds) + vid = torch.from_numpy(np.array(vid)).permute(0, 3, 1, 2) + vid = torch.unsqueeze(vid, dim=0) + self.writer.add_video( + vid, tag="{}: {}".format(tag, true_class_name) + ) + + mask = self._pick_wrong_preds(labels, preds) + video_indices = torch.squeeze(mask.nonzero(), dim=-1) + # Visualize each wrongly classfied video. + for vid_idx in video_indices: + cur_vid_idx = batch_idx * len(video_input[0]) + vid_idx + for pathway in range(len(video_input)): + add_video( + video_input[pathway][vid_idx], + preds=preds[vid_idx], + tag=self.tag + + "/Video {}, Pathway {}".format(cur_vid_idx, pathway), + true_class_name=self.class_names[labels[vid_idx]], + ) + + @property + def wrong_class_prediction(self): + """ + Return class ids that the model predicted incorrectly. + """ + incorrect_class_names = [ + self.class_names[i] for i in self.model_incorrect_classes + ] + return list(set(incorrect_class_names)) + + def clean(self): + """ + Close Tensorboard writer. + """ + self.writer.close() diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/predictor.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/predictor.py" new file mode 100644 index 0000000..443c8e4 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/predictor.py" @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import queue +import cv2 +import torch +from detectron2 import model_zoo +from detectron2.config import get_cfg +from detectron2.engine import DefaultPredictor + +import slowfast.utils.checkpoint as cu +from slowfast.datasets import cv2_transform +from slowfast.models import build_model +from slowfast.utils import logging +from slowfast.visualization.utils import process_cv2_inputs + +logger = logging.get_logger(__name__) + + +class Predictor: + """ + Action Predictor for action recognition. + """ + + def __init__(self, cfg, gpu_id=None): + """ + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + gpu_id (Optional[int]): GPU id. + """ + if cfg.NUM_GPUS: + self.gpu_id = ( + torch.cuda.current_device() if gpu_id is None else gpu_id + ) + + # Build the video model and print model statistics. + self.model = build_model(cfg, gpu_id=gpu_id) + self.model.eval() + self.cfg = cfg + + if cfg.DETECTION.ENABLE: + self.object_detector = Detectron2Predictor(cfg, gpu_id=self.gpu_id) + + logger.info("Start loading model weights.") + cu.load_test_checkpoint(cfg, self.model) + logger.info("Finish loading model weights") + + def __call__(self, task): + """ + Returns the prediction results for the current task. + Args: + task (TaskInfo object): task object that contain + the necessary information for action prediction. (e.g. frames, boxes) + Returns: + task (TaskInfo object): the same task info object but filled with + prediction values (a tensor) and the corresponding boxes for + action detection task. + """ + if self.cfg.DETECTION.ENABLE: + task = self.object_detector(task) + + frames, bboxes = task.frames, task.bboxes + if bboxes is not None: + bboxes = cv2_transform.scale_boxes( + self.cfg.DATA.TEST_CROP_SIZE, + bboxes, + task.img_height, + task.img_width, + ) + if self.cfg.DEMO.INPUT_FORMAT == "BGR": + frames = [ + cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) for frame in frames + ] + + frames = [ + cv2_transform.scale(self.cfg.DATA.TEST_CROP_SIZE, frame) + for frame in frames + ] + inputs = process_cv2_inputs(frames, self.cfg) + if bboxes is not None: + index_pad = torch.full( + size=(bboxes.shape[0], 1), + fill_value=float(0), + device=bboxes.device, + ) + + # Pad frame index for each box. + bboxes = torch.cat([index_pad, bboxes], axis=1) + if self.cfg.NUM_GPUS > 0: + # Transfer the data to the current GPU device. + if isinstance(inputs, (list,)): + for i in range(len(inputs)): + inputs[i] = inputs[i].cuda( + device=torch.device(self.gpu_id), non_blocking=True + ) + else: + inputs = inputs.cuda( + device=torch.device(self.gpu_id), non_blocking=True + ) + if self.cfg.DETECTION.ENABLE and not bboxes.shape[0]: + preds = torch.tensor([]) + else: + preds = self.model(inputs, bboxes) + + if self.cfg.NUM_GPUS: + preds = preds.cpu() + if bboxes is not None: + bboxes = bboxes.detach().cpu() + + preds = preds.detach() + task.add_action_preds(preds) + if bboxes is not None: + task.add_bboxes(bboxes[:, 1:]) + + return task + + +class ActionPredictor: + """ + Synchronous Action Prediction and Visualization pipeline with AsyncVis. + """ + + def __init__(self, cfg, async_vis=None, gpu_id=None): + """ + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + async_vis (AsyncVis object): asynchronous visualizer. + gpu_id (Optional[int]): GPU id. + """ + self.predictor = Predictor(cfg=cfg, gpu_id=gpu_id) + self.async_vis = async_vis + + def put(self, task): + """ + Make prediction and put the results in `async_vis` task queue. + Args: + task (TaskInfo object): task object that contain + the necessary information for action prediction. (e.g. frames, boxes) + """ + task = self.predictor(task) + self.async_vis.get_indices_ls.append(task.id) + self.async_vis.put(task) + + def get(self): + """ + Get the visualized clips if any. + """ + try: + task = self.async_vis.get() + except (queue.Empty, IndexError): + raise IndexError("Results are not available yet.") + + return task + + +class Detectron2Predictor: + """ + Wrapper around Detectron2 to return the required predicted bounding boxes + as a ndarray. + """ + + def __init__(self, cfg, gpu_id=None): + """ + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + gpu_id (Optional[int]): GPU id. + """ + + self.cfg = get_cfg() + self.cfg.merge_from_file( + model_zoo.get_config_file(cfg.DEMO.DETECTRON2_CFG) + ) + self.cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = cfg.DEMO.DETECTRON2_THRESH + self.cfg.MODEL.WEIGHTS = cfg.DEMO.DETECTRON2_WEIGHTS + self.cfg.INPUT.FORMAT = cfg.DEMO.INPUT_FORMAT + if cfg.NUM_GPUS and gpu_id is None: + gpu_id = torch.cuda.current_device() + self.cfg.MODEL.DEVICE = ( + "cuda:{}".format(gpu_id) if cfg.NUM_GPUS > 0 else "cpu" + ) + + logger.info("Initialized Detectron2 Object Detection Model.") + + self.predictor = DefaultPredictor(self.cfg) + + def __call__(self, task): + """ + Return bounding boxes predictions as a tensor. + Args: + task (TaskInfo object): task object that contain + the necessary information for action prediction. (e.g. frames) + Returns: + task (TaskInfo object): the same task info object but filled with + prediction values (a tensor) and the corresponding boxes for + action detection task. + """ + middle_frame = task.frames[len(task.frames) // 2] + outputs = self.predictor(middle_frame) + # Get only human instances + mask = outputs["instances"].pred_classes == 0 + pred_boxes = outputs["instances"].pred_boxes.tensor[mask] + task.add_bboxes(pred_boxes) + + return task diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/tensorboard_vis.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/tensorboard_vis.py" new file mode 100644 index 0000000..9d928c5 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/tensorboard_vis.py" @@ -0,0 +1,429 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import logging as log +import math +import os +import matplotlib.pyplot as plt +import torch +from torch.utils.tensorboard import SummaryWriter +from torchvision.utils import make_grid + +import slowfast.utils.logging as logging +import slowfast.visualization.utils as vis_utils +from slowfast.utils.misc import get_class_names + +logger = logging.get_logger(__name__) +log.getLogger("matplotlib").setLevel(log.ERROR) + + +class TensorboardWriter(object): + """ + Helper class to log information to Tensorboard. + """ + + def __init__(self, cfg): + """ + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + # class_names: list of class names. + # cm_subset_classes: a list of class ids -- a user-specified subset. + # parent_map: dictionary where key is the parent class name and + # value is a list of ids of its children classes. + # hist_subset_classes: a list of class ids -- user-specified to plot histograms. + ( + self.class_names, + self.cm_subset_classes, + self.parent_map, + self.hist_subset_classes, + ) = (None, None, None, None) + self.cfg = cfg + self.cm_figsize = cfg.TENSORBOARD.CONFUSION_MATRIX.FIGSIZE + self.hist_figsize = cfg.TENSORBOARD.HISTOGRAM.FIGSIZE + + if cfg.TENSORBOARD.LOG_DIR == "": + log_dir = os.path.join( + cfg.OUTPUT_DIR, "runs-{}".format(cfg.TRAIN.DATASET) + ) + else: + log_dir = os.path.join(cfg.OUTPUT_DIR, cfg.TENSORBOARD.LOG_DIR) + + self.writer = SummaryWriter(log_dir=log_dir) + logger.info( + "To see logged results in Tensorboard, please launch using the command \ + `tensorboard --port= --logdir {}`".format( + log_dir + ) + ) + + if cfg.TENSORBOARD.CLASS_NAMES_PATH != "": + if cfg.DETECTION.ENABLE: + logger.info( + "Plotting confusion matrix is currently \ + not supported for detection." + ) + ( + self.class_names, + self.parent_map, + self.cm_subset_classes, + ) = get_class_names( + cfg.TENSORBOARD.CLASS_NAMES_PATH, + cfg.TENSORBOARD.CATEGORIES_PATH, + cfg.TENSORBOARD.CONFUSION_MATRIX.SUBSET_PATH, + ) + + if cfg.TENSORBOARD.HISTOGRAM.ENABLE: + if cfg.DETECTION.ENABLE: + logger.info( + "Plotting histogram is not currently \ + supported for detection tasks." + ) + if cfg.TENSORBOARD.HISTOGRAM.SUBSET_PATH != "": + _, _, self.hist_subset_classes = get_class_names( + cfg.TENSORBOARD.CLASS_NAMES_PATH, + None, + cfg.TENSORBOARD.HISTOGRAM.SUBSET_PATH, + ) + + def add_scalars(self, data_dict, global_step=None): + """ + Add multiple scalars to Tensorboard logs. + Args: + data_dict (dict): key is a string specifying the tag of value. + global_step (Optinal[int]): Global step value to record. + """ + if self.writer is not None: + for key, item in data_dict.items(): + self.writer.add_scalar(key, item, global_step) + + def plot_eval(self, preds, labels, global_step=None): + """ + Plot confusion matrices and histograms for eval/test set. + Args: + preds (tensor or list of tensors): list of predictions. + labels (tensor or list of tensors): list of labels. + global step (Optional[int]): current step in eval/test. + """ + if not self.cfg.DETECTION.ENABLE: + cmtx = None + if self.cfg.TENSORBOARD.CONFUSION_MATRIX.ENABLE: + cmtx = vis_utils.get_confusion_matrix( + preds, labels, self.cfg.MODEL.NUM_CLASSES + ) + # Add full confusion matrix. + add_confusion_matrix( + self.writer, + cmtx, + self.cfg.MODEL.NUM_CLASSES, + global_step=global_step, + class_names=self.class_names, + figsize=self.cm_figsize, + ) + # If a list of subset is provided, plot confusion matrix subset. + if self.cm_subset_classes is not None: + add_confusion_matrix( + self.writer, + cmtx, + self.cfg.MODEL.NUM_CLASSES, + global_step=global_step, + subset_ids=self.cm_subset_classes, + class_names=self.class_names, + tag="Confusion Matrix Subset", + figsize=self.cm_figsize, + ) + # If a parent-child classes mapping is provided, plot confusion + # matrices grouped by parent classes. + if self.parent_map is not None: + # Get list of tags (parent categories names) and their children. + for parent_class, children_ls in self.parent_map.items(): + tag = ( + "Confusion Matrices Grouped by Parent Classes/" + + parent_class + ) + add_confusion_matrix( + self.writer, + cmtx, + self.cfg.MODEL.NUM_CLASSES, + global_step=global_step, + subset_ids=children_ls, + class_names=self.class_names, + tag=tag, + figsize=self.cm_figsize, + ) + if self.cfg.TENSORBOARD.HISTOGRAM.ENABLE: + if cmtx is None: + cmtx = vis_utils.get_confusion_matrix( + preds, labels, self.cfg.MODEL.NUM_CLASSES + ) + plot_hist( + self.writer, + cmtx, + self.cfg.MODEL.NUM_CLASSES, + self.cfg.TENSORBOARD.HISTOGRAM.TOPK, + global_step=global_step, + subset_ids=self.hist_subset_classes, + class_names=self.class_names, + figsize=self.hist_figsize, + ) + + def add_video(self, vid_tensor, tag="Video Input", global_step=None, fps=4): + """ + Add input to tensorboard SummaryWriter as a video. + Args: + vid_tensor (tensor): shape of (B, T, C, H, W). Values should lie + [0, 255] for type uint8 or [0, 1] for type float. + tag (Optional[str]): name of the video. + global_step(Optional[int]): current step. + fps (int): frames per second. + """ + self.writer.add_video(tag, vid_tensor, global_step=global_step, fps=fps) + + def plot_weights_and_activations( + self, + weight_activation_dict, + tag="", + normalize=False, + global_step=None, + batch_idx=None, + indexing_dict=None, + heat_map=True, + ): + """ + Visualize weights/ activations tensors to Tensorboard. + Args: + weight_activation_dict (dict[str, tensor]): a dictionary of the pair {layer_name: tensor}, + where layer_name is a string and tensor is the weights/activations of + the layer we want to visualize. + tag (Optional[str]): name of the video. + normalize (bool): If True, the tensor is normalized. (Default to False) + global_step(Optional[int]): current step. + batch_idx (Optional[int]): current batch index to visualize. If None, + visualize the entire batch. + indexing_dict (Optional[dict]): a dictionary of the {layer_name: indexing}. + where indexing is numpy-like fancy indexing. + heatmap (bool): whether to add heatmap to the weights/ activations. + """ + for name, array in weight_activation_dict.items(): + if batch_idx is None: + # Select all items in the batch if batch_idx is not provided. + batch_idx = list(range(array.shape[0])) + if indexing_dict is not None: + fancy_indexing = indexing_dict[name] + fancy_indexing = (batch_idx,) + fancy_indexing + array = array[fancy_indexing] + else: + array = array[batch_idx] + add_ndim_array( + self.writer, + array, + tag + name, + normalize=normalize, + global_step=global_step, + heat_map=heat_map, + ) + + def flush(self): + self.writer.flush() + + def close(self): + self.writer.flush() + self.writer.close() + + +def add_confusion_matrix( + writer, + cmtx, + num_classes, + global_step=None, + subset_ids=None, + class_names=None, + tag="Confusion Matrix", + figsize=None, +): + """ + Calculate and plot confusion matrix to a SummaryWriter. + Args: + writer (SummaryWriter): the SummaryWriter to write the matrix to. + cmtx (ndarray): confusion matrix. + num_classes (int): total number of classes. + global_step (Optional[int]): current step. + subset_ids (list of ints): a list of label indices to keep. + class_names (list of strs, optional): a list of all class names. + tag (str or list of strs): name(s) of the confusion matrix image. + figsize (Optional[float, float]): the figure size of the confusion matrix. + If None, default to [6.4, 4.8]. + + """ + if subset_ids is None or len(subset_ids) != 0: + # If class names are not provided, use class indices as class names. + if class_names is None: + class_names = [str(i) for i in range(num_classes)] + # If subset is not provided, take every classes. + if subset_ids is None: + subset_ids = list(range(num_classes)) + + sub_cmtx = cmtx[subset_ids, :][:, subset_ids] + sub_names = [class_names[j] for j in subset_ids] + + sub_cmtx = vis_utils.plot_confusion_matrix( + sub_cmtx, + num_classes=len(subset_ids), + class_names=sub_names, + figsize=figsize, + ) + # Add the confusion matrix image to writer. + writer.add_figure(tag=tag, figure=sub_cmtx, global_step=global_step) + + +def plot_hist( + writer, + cmtx, + num_classes, + k=10, + global_step=None, + subset_ids=None, + class_names=None, + figsize=None, +): + """ + Given all predictions and all true labels, plot histograms of top-k most + frequently predicted classes for each true class. + + Args: + writer (SummaryWriter object): a tensorboard SummaryWriter object. + cmtx (ndarray): confusion matrix. + num_classes (int): total number of classes. + k (int): top k to plot histograms. + global_step (Optional[int]): current step. + subset_ids (list of ints, optional): class indices to plot histogram. + mapping (list of strings): names of all classes. + figsize (Optional[float, float]): the figure size of the confusion matrix. + If None, default to [6.4, 4.8]. + """ + if subset_ids is None or len(subset_ids) != 0: + if subset_ids is None: + subset_ids = set(range(num_classes)) + else: + subset_ids = set(subset_ids) + # If class names are not provided, use their indices as names. + if class_names is None: + class_names = list(range(num_classes)) + + for i in subset_ids: + pred = cmtx[i] + hist = vis_utils.plot_topk_histogram( + class_names[i], + torch.Tensor(pred), + k, + class_names, + figsize=figsize, + ) + writer.add_figure( + tag="Top {} predictions by classes/{}".format( + k, class_names[i] + ), + figure=hist, + global_step=global_step, + ) + + +def add_ndim_array( + writer, + array, + name, + nrow=None, + normalize=False, + global_step=None, + heat_map=True, +): + """ + Visualize and add tensors of n-dimentionals to a Tensorboard SummaryWriter. Tensors + will be visualized as a 2D grid image. + Args: + writer (SummaryWriter): Tensorboard SummaryWriter. + array (tensor): tensor to visualize. + name (str): name of the tensor. + nrow (Optional[int]): number of 2D filters in each row in the grid image. + normalize (bool): whether to normalize when we have multiple 2D filters. + Default to False. + global_step (Optional[int]): current step. + heat_map (bool): whether to add heat map to 2D each 2D filters in array. + """ + if array is not None and array.ndim != 0: + if array.ndim == 1: + reshaped_array = array.unsqueeze(0) + if nrow is None: + nrow = int(math.sqrt(reshaped_array.size()[1])) + reshaped_array = reshaped_array.view(-1, nrow) + if heat_map: + reshaped_array = add_heatmap(reshaped_array) + writer.add_image( + name, + reshaped_array, + global_step=global_step, + dataformats="CHW", + ) + else: + writer.add_image( + name, + reshaped_array, + global_step=global_step, + dataformats="HW", + ) + elif array.ndim == 2: + reshaped_array = array + if heat_map: + heatmap = add_heatmap(reshaped_array) + writer.add_image( + name, heatmap, global_step=global_step, dataformats="CHW" + ) + else: + writer.add_image( + name, + reshaped_array, + global_step=global_step, + dataformats="HW", + ) + else: + last2_dims = array.size()[-2:] + reshaped_array = array.view(-1, *last2_dims) + if heat_map: + reshaped_array = [ + add_heatmap(array_2d).unsqueeze(0) + for array_2d in reshaped_array + ] + reshaped_array = torch.cat(reshaped_array, dim=0) + else: + reshaped_array = reshaped_array.unsqueeze(1) + if nrow is None: + nrow = int(math.sqrt(reshaped_array.size()[0])) + img_grid = make_grid( + reshaped_array, nrow, padding=1, normalize=normalize + ) + writer.add_image(name, img_grid, global_step=global_step) + + +def add_heatmap(tensor): + """ + Add heatmap to 2D tensor. + Args: + tensor (tensor): a 2D tensor. Tensor value must be in [0..1] range. + Returns: + heatmap (tensor): a 3D tensor. Result of applying heatmap to the 2D tensor. + """ + assert tensor.ndim == 2, "Only support 2D tensors." + # Move tensor to cpu if necessary. + if tensor.device != torch.device("cpu"): + arr = tensor.cpu() + else: + arr = tensor + arr = arr.numpy() + # Get the color map by name. + cm = plt.get_cmap("viridis") + heatmap = cm(arr) + heatmap = heatmap[:, :, :3] + # Convert (H, W, C) to (C, H, W) + heatmap = torch.Tensor(heatmap).permute(2, 0, 1) + return heatmap diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/utils.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/utils.py" new file mode 100644 index 0000000..a07be13 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/utils.py" @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import itertools +import numpy as np +import matplotlib.pyplot as plt +import torch +from sklearn.metrics import confusion_matrix + +import slowfast.utils.logging as logging +from slowfast.datasets.utils import pack_pathway_output, tensor_normalize + +logger = logging.get_logger(__name__) + + +def get_confusion_matrix(preds, labels, num_classes, normalize="true"): + """ + Calculate confusion matrix on the provided preds and labels. + Args: + preds (tensor or lists of tensors): predictions. Each tensor is in + in the shape of (n_batch, num_classes). Tensor(s) must be on CPU. + labels (tensor or lists of tensors): corresponding labels. Each tensor is + in the shape of either (n_batch,) or (n_batch, num_classes). + num_classes (int): number of classes. Tensor(s) must be on CPU. + normalize (Optional[str]) : {‘true’, ‘pred’, ‘all’}, default="true" + Normalizes confusion matrix over the true (rows), predicted (columns) + conditions or all the population. If None, confusion matrix + will not be normalized. + Returns: + cmtx (ndarray): confusion matrix of size (num_classes x num_classes) + """ + if isinstance(preds, list): + preds = torch.cat(preds, dim=0) + if isinstance(labels, list): + labels = torch.cat(labels, dim=0) + # If labels are one-hot encoded, get their indices. + if labels.ndim == preds.ndim: + labels = torch.argmax(labels, dim=-1) + # Get the predicted class indices for examples. + preds = torch.flatten(torch.argmax(preds, dim=-1)) + labels = torch.flatten(labels) + cmtx = confusion_matrix( + labels, preds, labels=list(range(num_classes)), normalize=normalize + ) + return cmtx + + +def plot_confusion_matrix(cmtx, num_classes, class_names=None, figsize=None): + """ + A function to create a colored and labeled confusion matrix matplotlib figure + given true labels and preds. + Args: + cmtx (ndarray): confusion matrix. + num_classes (int): total number of classes. + class_names (Optional[list of strs]): a list of class names. + figsize (Optional[float, float]): the figure size of the confusion matrix. + If None, default to [6.4, 4.8]. + + Returns: + img (figure): matplotlib figure. + """ + if class_names is None or type(class_names) != list: + class_names = [str(i) for i in range(num_classes)] + + figure = plt.figure(figsize=figsize) + plt.imshow(cmtx, interpolation="nearest", cmap=plt.cm.Blues) + plt.title("Confusion matrix") + plt.colorbar() + tick_marks = np.arange(len(class_names)) + plt.xticks(tick_marks, class_names, rotation=45) + plt.yticks(tick_marks, class_names) + + # Use white text if squares are dark; otherwise black. + threshold = cmtx.max() / 2.0 + for i, j in itertools.product(range(cmtx.shape[0]), range(cmtx.shape[1])): + color = "white" if cmtx[i, j] > threshold else "black" + plt.text( + j, + i, + format(cmtx[i, j], ".2f") if cmtx[i, j] != 0 else ".", + horizontalalignment="center", + color=color, + ) + + plt.tight_layout() + plt.ylabel("True label") + plt.xlabel("Predicted label") + + return figure + + +def plot_topk_histogram(tag, array, k=10, class_names=None, figsize=None): + """ + Plot histogram of top-k value from the given array. + Args: + tag (str): histogram title. + array (tensor): a tensor to draw top k value from. + k (int): number of top values to draw from array. + Defaut to 10. + class_names (list of strings, optional): + a list of names for values in array. + figsize (Optional[float, float]): the figure size of the confusion matrix. + If None, default to [6.4, 4.8]. + Returns: + fig (matplotlib figure): a matplotlib figure of the histogram. + """ + val, ind = torch.topk(array, k) + + fig = plt.Figure(figsize=figsize, facecolor="w", edgecolor="k") + + ax = fig.add_subplot(1, 1, 1) + + if class_names is None: + class_names = [str(i) for i in ind] + else: + class_names = [class_names[i] for i in ind] + + tick_marks = np.arange(k) + width = 0.75 + ax.bar( + tick_marks, + val, + width, + color="orange", + tick_label=class_names, + edgecolor="w", + linewidth=1, + ) + + ax.set_xlabel("Candidates") + ax.set_xticks(tick_marks) + ax.set_xticklabels(class_names, rotation=-45, ha="center") + ax.xaxis.set_label_position("bottom") + ax.xaxis.tick_bottom() + + y_tick = np.linspace(0, 1, num=10) + ax.set_ylabel("Frequency") + ax.set_yticks(y_tick) + y_labels = [format(i, ".1f") for i in y_tick] + ax.set_yticklabels(y_labels, ha="center") + + for i, v in enumerate(val.numpy()): + ax.text( + i - 0.1, + v + 0.03, + format(v, ".2f"), + color="orange", + fontweight="bold", + ) + + ax.set_title(tag) + + fig.set_tight_layout(True) + + return fig + + +class GetWeightAndActivation: + """ + A class used to get weights and activations from specified layers from a Pytorch model. + """ + + def __init__(self, model, layers): + """ + Args: + model (nn.Module): the model containing layers to obtain weights and activations from. + layers (list of strings): a list of layer names to obtain weights and activations from. + Names are hierarchical, separated by /. For example, If a layer follow a path + "s1" ---> "pathway0_stem" ---> "conv", the layer path is "s1/pathway0_stem/conv". + """ + self.model = model + self.hooks = {} + self.layers_names = layers + # eval mode + self.model.eval() + self._register_hooks() + + def _get_layer(self, layer_name): + """ + Return a layer (nn.Module Object) given a hierarchical layer name, separated by /. + Args: + layer_name (str): the name of the layer. + """ + layer_ls = layer_name.split("/") + prev_module = self.model + for layer in layer_ls: + prev_module = prev_module._modules[layer] + + return prev_module + + def _register_single_hook(self, layer_name): + """ + Register hook to a layer, given layer_name, to obtain activations. + Args: + layer_name (str): name of the layer. + """ + + def hook_fn(module, input, output): + self.hooks[layer_name] = output.clone().detach() + + layer = get_layer(self.model, layer_name) + layer.register_forward_hook(hook_fn) + + def _register_hooks(self): + """ + Register hooks to layers in `self.layers_names`. + """ + for layer_name in self.layers_names: + self._register_single_hook(layer_name) + + def get_activations(self, input, bboxes=None): + """ + Obtain all activations from layers that we register hooks for. + Args: + input (tensors, list of tensors): the model input. + bboxes (Optional): Bouding boxes data that might be required + by the model. + Returns: + activation_dict (Python dictionary): a dictionary of the pair + {layer_name: list of activations}, where activations are outputs returned + by the layer. + """ + input_clone = [inp.clone() for inp in input] + if bboxes is not None: + preds = self.model(input_clone, bboxes) + else: + preds = self.model(input_clone) + + activation_dict = {} + for layer_name, hook in self.hooks.items(): + # list of activations for each instance. + activation_dict[layer_name] = hook + + return activation_dict, preds + + def get_weights(self): + """ + Returns weights from registered layers. + Returns: + weights (Python dictionary): a dictionary of the pair + {layer_name: weight}, where weight is the weight tensor. + """ + weights = {} + for layer in self.layers_names: + cur_layer = get_layer(self.model, layer) + if hasattr(cur_layer, "weight"): + weights[layer] = cur_layer.weight.clone().detach() + else: + logger.error( + "Layer {} does not have weight attribute.".format(layer) + ) + return weights + + +def get_indexing(string): + """ + Parse numpy-like fancy indexing from a string. + Args: + string (str): string represent the indices to take + a subset of from array. Indices for each dimension + are separated by `,`; indices for different dimensions + are separated by `;`. + e.g.: For a numpy array `arr` of shape (3,3,3), the string "1,2;1,2" + means taking the sub-array `arr[[1,2], [1,2]] + Returns: + final_indexing (tuple): the parsed indexing. + """ + index_ls = string.strip().split(";") + final_indexing = [] + for index in index_ls: + index_single_dim = index.split(",") + index_single_dim = [int(i) for i in index_single_dim] + final_indexing.append(index_single_dim) + + return tuple(final_indexing) + + +def process_layer_index_data(layer_ls, layer_name_prefix=""): + """ + Extract layer names and numpy-like fancy indexing from a string. + Args: + layer_ls (list of strs): list of strings containing data about layer names + and their indexing. For each string, layer name and indexing is separated by whitespaces. + e.g.: [layer1 1,2;2, layer2, layer3 150;3,4] + layer_name_prefix (Optional[str]): prefix to be added to each layer name. + Returns: + layer_name (list of strings): a list of layer names. + indexing_dict (Python dict): a dictionary of the pair + {one_layer_name: indexing_for_that_layer} + """ + + layer_name, indexing_dict = [], {} + for layer in layer_ls: + ls = layer.split() + name = layer_name_prefix + ls[0] + layer_name.append(name) + if len(ls) == 2: + indexing_dict[name] = get_indexing(ls[1]) + else: + indexing_dict[name] = () + return layer_name, indexing_dict + + +def process_cv2_inputs(frames, cfg): + """ + Normalize and prepare inputs as a list of tensors. Each tensor + correspond to a unique pathway. + Args: + frames (list of array): list of input images (correspond to one clip) in range [0, 255]. + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + inputs = torch.from_numpy(np.array(frames)).float() / 255 + inputs = tensor_normalize(inputs, cfg.DATA.MEAN, cfg.DATA.STD) + # T H W C -> C T H W. + inputs = inputs.permute(3, 0, 1, 2) + # Sample frames for num_frames specified. + index = torch.linspace(0, inputs.shape[1] - 1, cfg.DATA.NUM_FRAMES).long() + inputs = torch.index_select(inputs, 1, index) + inputs = pack_pathway_output(cfg, inputs) + inputs = [inp.unsqueeze(0) for inp in inputs] + return inputs + + +def get_layer(model, layer_name): + """ + Return the targeted layer (nn.Module Object) given a hierarchical layer name, + separated by /. + Args: + model (model): model to get layers from. + layer_name (str): name of the layer. + Returns: + prev_module (nn.Module): the layer from the model with `layer_name` name. + """ + layer_ls = layer_name.split("/") + prev_module = model + for layer in layer_ls: + prev_module = prev_module._modules[layer] + + return prev_module + + +class TaskInfo: + def __init__(self): + self.frames = None + self.id = -1 + self.bboxes = None + self.action_preds = None + self.num_buffer_frames = 0 + self.img_height = -1 + self.img_width = -1 + self.crop_size = -1 + self.clip_vis_size = -1 + + def add_frames(self, idx, frames): + """ + Add the clip and corresponding id. + Args: + idx (int): the current index of the clip. + frames (list[ndarray]): list of images in "BGR" format. + """ + self.frames = frames + self.id = idx + + def add_bboxes(self, bboxes): + """ + Add correspondding bounding boxes. + """ + self.bboxes = bboxes + + def add_action_preds(self, preds): + """ + Add the corresponding action predictions. + """ + self.action_preds = preds diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/video_visualizer.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/video_visualizer.py" new file mode 100644 index 0000000..faa1272 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/slowfast/visualization/video_visualizer.py" @@ -0,0 +1,677 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import itertools +import logging as log +import numpy as np +import matplotlib.pyplot as plt +import torch +from detectron2.utils.visualizer import Visualizer + +import slowfast.utils.logging as logging +from slowfast.utils.misc import get_class_names + +logger = logging.get_logger(__name__) +log.getLogger("matplotlib").setLevel(log.ERROR) + + +def _create_text_labels(classes, scores, class_names, ground_truth=False): + """ + Create text labels. + Args: + classes (list[int]): a list of class ids for each example. + scores (list[float] or None): list of scores for each example. + class_names (list[str]): a list of class names, ordered by their ids. + ground_truth (bool): whether the labels are ground truth. + Returns: + labels (list[str]): formatted text labels. + """ + try: + labels = [class_names[i] for i in classes] + except IndexError: + logger.error("Class indices get out of range: {}".format(classes)) + return None + + if ground_truth: + labels = ["[{}] {}".format("GT", label) for label in labels] + elif scores is not None: + assert len(classes) == len(scores) + labels = [ + "[{:.2f}] {}".format(s, label) for s, label in zip(scores, labels) + ] + return labels + + +class ImgVisualizer(Visualizer): + def __init__(self, img_rgb, meta, **kwargs): + """ + See https://github.com/facebookresearch/detectron2/blob/master/detectron2/utils/visualizer.py + for more details. + Args: + img_rgb: a tensor or numpy array of shape (H, W, C), where H and W correspond to + the height and width of the image respectively. C is the number of + color channels. The image is required to be in RGB format since that + is a requirement of the Matplotlib library. The image is also expected + to be in the range [0, 255]. + meta (MetadataCatalog): image metadata. + See https://github.com/facebookresearch/detectron2/blob/81d5a87763bfc71a492b5be89b74179bd7492f6b/detectron2/data/catalog.py#L90 + """ + super(ImgVisualizer, self).__init__(img_rgb, meta, **kwargs) + + def draw_text( + self, + text, + position, + *, + font_size=None, + color="w", + horizontal_alignment="center", + vertical_alignment="bottom", + box_facecolor="black", + alpha=0.5, + ): + """ + Draw text at the specified position. + Args: + text (str): the text to draw on image. + position (list of 2 ints): the x,y coordinate to place the text. + font_size (Optional[int]): font of the text. If not provided, a font size + proportional to the image width is calculated and used. + color (str): color of the text. Refer to `matplotlib.colors` for full list + of formats that are accepted. + horizontal_alignment (str): see `matplotlib.text.Text`. + vertical_alignment (str): see `matplotlib.text.Text`. + box_facecolor (str): color of the box wrapped around the text. Refer to + `matplotlib.colors` for full list of formats that are accepted. + alpha (float): transparency level of the box. + """ + if not font_size: + font_size = self._default_font_size + x, y = position + self.output.ax.text( + x, + y, + text, + size=font_size * self.output.scale, + family="monospace", + bbox={ + "facecolor": box_facecolor, + "alpha": alpha, + "pad": 0.7, + "edgecolor": "none", + }, + verticalalignment=vertical_alignment, + horizontalalignment=horizontal_alignment, + color=color, + zorder=10, + ) + + def draw_multiple_text( + self, + text_ls, + box_coordinate, + *, + top_corner=True, + font_size=None, + color="w", + box_facecolors="black", + alpha=0.5, + ): + """ + Draw a list of text labels for some bounding box on the image. + Args: + text_ls (list of strings): a list of text labels. + box_coordinate (tensor): shape (4,). The (x_left, y_top, x_right, y_bottom) + coordinates of the box. + top_corner (bool): If True, draw the text labels at (x_left, y_top) of the box. + Else, draw labels at (x_left, y_bottom). + font_size (Optional[int]): font of the text. If not provided, a font size + proportional to the image width is calculated and used. + color (str): color of the text. Refer to `matplotlib.colors` for full list + of formats that are accepted. + box_facecolors (str): colors of the box wrapped around the text. Refer to + `matplotlib.colors` for full list of formats that are accepted. + alpha (float): transparency level of the box. + """ + if not isinstance(box_facecolors, list): + box_facecolors = [box_facecolors] * len(text_ls) + assert len(box_facecolors) == len( + text_ls + ), "Number of colors provided is not equal to the number of text labels." + if not font_size: + font_size = self._default_font_size + text_box_width = font_size + font_size // 2 + # If the texts does not fit in the assigned location, + # we split the text and draw it in another place. + if top_corner: + num_text_split = self._align_y_top( + box_coordinate, len(text_ls), text_box_width + ) + y_corner = 1 + else: + num_text_split = len(text_ls) - self._align_y_bottom( + box_coordinate, len(text_ls), text_box_width + ) + y_corner = 3 + + text_color_sorted = sorted( + zip(text_ls, box_facecolors), key=lambda x: x[0], reverse=True + ) + if len(text_color_sorted) != 0: + text_ls, box_facecolors = zip(*text_color_sorted) + else: + text_ls, box_facecolors = [], [] + text_ls, box_facecolors = list(text_ls), list(box_facecolors) + self.draw_multiple_text_upward( + text_ls[:num_text_split][::-1], + box_coordinate, + y_corner=y_corner, + font_size=font_size, + color=color, + box_facecolors=box_facecolors[:num_text_split][::-1], + alpha=alpha, + ) + self.draw_multiple_text_downward( + text_ls[num_text_split:], + box_coordinate, + y_corner=y_corner, + font_size=font_size, + color=color, + box_facecolors=box_facecolors[num_text_split:], + alpha=alpha, + ) + + def draw_multiple_text_upward( + self, + text_ls, + box_coordinate, + *, + y_corner=1, + font_size=None, + color="w", + box_facecolors="black", + alpha=0.5, + ): + """ + Draw a list of text labels for some bounding box on the image in upward direction. + The next text label will be on top of the previous one. + Args: + text_ls (list of strings): a list of text labels. + box_coordinate (tensor): shape (4,). The (x_left, y_top, x_right, y_bottom) + coordinates of the box. + y_corner (int): Value of either 1 or 3. Indicate the index of the y-coordinate of + the box to draw labels around. + font_size (Optional[int]): font of the text. If not provided, a font size + proportional to the image width is calculated and used. + color (str): color of the text. Refer to `matplotlib.colors` for full list + of formats that are accepted. + box_facecolors (str or list of strs): colors of the box wrapped around the text. Refer to + `matplotlib.colors` for full list of formats that are accepted. + alpha (float): transparency level of the box. + """ + if not isinstance(box_facecolors, list): + box_facecolors = [box_facecolors] * len(text_ls) + assert len(box_facecolors) == len( + text_ls + ), "Number of colors provided is not equal to the number of text labels." + + assert y_corner in [1, 3], "Y_corner must be either 1 or 3" + if not font_size: + font_size = self._default_font_size + + x, horizontal_alignment = self._align_x_coordinate(box_coordinate) + y = box_coordinate[y_corner].item() + for i, text in enumerate(text_ls): + self.draw_text( + text, + (x, y), + font_size=font_size, + color=color, + horizontal_alignment=horizontal_alignment, + vertical_alignment="bottom", + box_facecolor=box_facecolors[i], + alpha=alpha, + ) + y -= font_size + font_size // 2 + + def draw_multiple_text_downward( + self, + text_ls, + box_coordinate, + *, + y_corner=1, + font_size=None, + color="w", + box_facecolors="black", + alpha=0.5, + ): + """ + Draw a list of text labels for some bounding box on the image in downward direction. + The next text label will be below the previous one. + Args: + text_ls (list of strings): a list of text labels. + box_coordinate (tensor): shape (4,). The (x_left, y_top, x_right, y_bottom) + coordinates of the box. + y_corner (int): Value of either 1 or 3. Indicate the index of the y-coordinate of + the box to draw labels around. + font_size (Optional[int]): font of the text. If not provided, a font size + proportional to the image width is calculated and used. + color (str): color of the text. Refer to `matplotlib.colors` for full list + of formats that are accepted. + box_facecolors (str): colors of the box wrapped around the text. Refer to + `matplotlib.colors` for full list of formats that are accepted. + alpha (float): transparency level of the box. + """ + if not isinstance(box_facecolors, list): + box_facecolors = [box_facecolors] * len(text_ls) + assert len(box_facecolors) == len( + text_ls + ), "Number of colors provided is not equal to the number of text labels." + + assert y_corner in [1, 3], "Y_corner must be either 1 or 3" + if not font_size: + font_size = self._default_font_size + + x, horizontal_alignment = self._align_x_coordinate(box_coordinate) + y = box_coordinate[y_corner].item() + for i, text in enumerate(text_ls): + self.draw_text( + text, + (x, y), + font_size=font_size, + color=color, + horizontal_alignment=horizontal_alignment, + vertical_alignment="top", + box_facecolor=box_facecolors[i], + alpha=alpha, + ) + y += font_size + font_size // 2 + + def _align_x_coordinate(self, box_coordinate): + """ + Choose an x-coordinate from the box to make sure the text label + does not go out of frames. By default, the left x-coordinate is + chosen and text is aligned left. If the box is too close to the + right side of the image, then the right x-coordinate is chosen + instead and the text is aligned right. + Args: + box_coordinate (array-like): shape (4,). The (x_left, y_top, x_right, y_bottom) + coordinates of the box. + Returns: + x_coordinate (float): the chosen x-coordinate. + alignment (str): whether to align left or right. + """ + # If the x-coordinate is greater than 5/6 of the image width, + # then we align test to the right of the box. This is + # chosen by heuristics. + if box_coordinate[0] > (self.output.width * 5) // 6: + return box_coordinate[2], "right" + + return box_coordinate[0], "left" + + def _align_y_top(self, box_coordinate, num_text, textbox_width): + """ + Calculate the number of text labels to plot on top of the box + without going out of frames. + Args: + box_coordinate (array-like): shape (4,). The (x_left, y_top, x_right, y_bottom) + coordinates of the box. + num_text (int): the number of text labels to plot. + textbox_width (float): the width of the box wrapped around text label. + """ + dist_to_top = box_coordinate[1] + num_text_top = dist_to_top // textbox_width + + if isinstance(num_text_top, torch.Tensor): + num_text_top = int(num_text_top.item()) + + return min(num_text, num_text_top) + + def _align_y_bottom(self, box_coordinate, num_text, textbox_width): + """ + Calculate the number of text labels to plot at the bottom of the box + without going out of frames. + Args: + box_coordinate (array-like): shape (4,). The (x_left, y_top, x_right, y_bottom) + coordinates of the box. + num_text (int): the number of text labels to plot. + textbox_width (float): the width of the box wrapped around text label. + """ + dist_to_bottom = self.output.height - box_coordinate[3] + num_text_bottom = dist_to_bottom // textbox_width + + if isinstance(num_text_bottom, torch.Tensor): + num_text_bottom = int(num_text_bottom.item()) + + return min(num_text, num_text_bottom) + + +class VideoVisualizer: + def __init__( + self, + num_classes, + class_names_path, + top_k=1, + colormap="rainbow", + thres=0.7, + lower_thres=0.3, + common_class_names=None, + mode="top-k", + ): + """ + Args: + num_classes (int): total number of classes. + class_names_path (str): path to json file that maps class names to ids. + Must be in the format {classname: id}. + top_k (int): number of top predicted classes to plot. + colormap (str): the colormap to choose color for class labels from. + See https://matplotlib.org/tutorials/colors/colormaps.html + thres (float): threshold for picking predicted classes to visualize. + lower_thres (Optional[float]): If `common_class_names` if given, + this `lower_thres` will be applied to uncommon classes and + `thres` will be applied to classes in `common_class_names`. + common_class_names (Optional[list of str(s)]): list of common class names + to apply `thres`. Class names not included in `common_class_names` will + have `lower_thres` as a threshold. If None, all classes will have `thres` as a threshold. + This is helpful for model trained on highly imbalanced dataset. + mode (str): Supported modes are {"top-k", "thres"}. + This is used for choosing predictions for visualization. + + """ + assert mode in ["top-k", "thres"], "Mode {} is not supported.".format( + mode + ) + self.mode = mode + self.num_classes = num_classes + self.class_names, _, _ = get_class_names(class_names_path, None, None) + self.top_k = top_k + self.thres = thres + self.lower_thres = lower_thres + + if mode == "thres": + self._get_thres_array(common_class_names=common_class_names) + + self.color_map = plt.get_cmap(colormap) + + def _get_color(self, class_id): + """ + Get color for a class id. + Args: + class_id (int): class id. + """ + return self.color_map(class_id / self.num_classes)[:3] + + def draw_one_frame( + self, + frame, + preds, + bboxes=None, + alpha=0.5, + text_alpha=0.7, + ground_truth=False, + ): + """ + Draw labels and bouding boxes for one image. By default, predicted labels are drawn in + the top left corner of the image or corresponding bounding boxes. For ground truth labels + (setting True for ground_truth flag), labels will be drawn in the bottom left corner. + Args: + frame (array-like): a tensor or numpy array of shape (H, W, C), where H and W correspond to + the height and width of the image respectively. C is the number of + color channels. The image is required to be in RGB format since that + is a requirement of the Matplotlib library. The image is also expected + to be in the range [0, 255]. + preds (tensor or list): If ground_truth is False, provide a float tensor of shape (num_boxes, num_classes) + that contains all of the confidence scores of the model. + For recognition task, input shape can be (num_classes,). To plot true label (ground_truth is True), + preds is a list contains int32 of the shape (num_boxes, true_class_ids) or (true_class_ids,). + bboxes (Optional[tensor]): shape (num_boxes, 4) that contains the coordinates of the bounding boxes. + alpha (Optional[float]): transparency level of the bounding boxes. + text_alpha (Optional[float]): transparency level of the box wrapped around text labels. + ground_truth (bool): whether the prodived bounding boxes are ground-truth. + """ + if isinstance(preds, torch.Tensor): + if preds.ndim == 1: + preds = preds.unsqueeze(0) + n_instances = preds.shape[0] + elif isinstance(preds, list): + n_instances = len(preds) + else: + logger.error("Unsupported type of prediction input.") + return + + if ground_truth: + top_scores, top_classes = [None] * n_instances, preds + + elif self.mode == "top-k": + top_scores, top_classes = torch.topk(preds, k=self.top_k) + top_scores, top_classes = top_scores.tolist(), top_classes.tolist() + elif self.mode == "thres": + top_scores, top_classes = [], [] + for pred in preds: + mask = pred >= self.thres + top_scores.append(pred[mask].tolist()) + top_class = torch.squeeze(torch.nonzero(mask), dim=-1).tolist() + top_classes.append(top_class) + + # Create labels top k predicted classes with their scores. + text_labels = [] + for i in range(n_instances): + text_labels.append( + _create_text_labels( + top_classes[i], + top_scores[i], + self.class_names, + ground_truth=ground_truth, + ) + ) + frame_visualizer = ImgVisualizer(frame, meta=None) + font_size = min( + max(np.sqrt(frame.shape[0] * frame.shape[1]) // 35, 5), 9 + ) + top_corner = not ground_truth + if bboxes is not None: + assert len(preds) == len( + bboxes + ), "Encounter {} predictions and {} bounding boxes".format( + len(preds), len(bboxes) + ) + for i, box in enumerate(bboxes): + text = text_labels[i] + pred_class = top_classes[i] + colors = [self._get_color(pred) for pred in pred_class] + + box_color = "r" if ground_truth else "g" + line_style = "--" if ground_truth else "-." + frame_visualizer.draw_box( + box, + alpha=alpha, + edge_color=box_color, + line_style=line_style, + ) + frame_visualizer.draw_multiple_text( + text, + box, + top_corner=top_corner, + font_size=font_size, + box_facecolors=colors, + alpha=text_alpha, + ) + else: + text = text_labels[0] + pred_class = top_classes[0] + colors = [self._get_color(pred) for pred in pred_class] + frame_visualizer.draw_multiple_text( + text, + torch.Tensor([0, 5, frame.shape[1], frame.shape[0] - 5]), + top_corner=top_corner, + font_size=font_size, + box_facecolors=colors, + alpha=text_alpha, + ) + + return frame_visualizer.output.get_image() + + def draw_clip_range( + self, + frames, + preds, + bboxes=None, + text_alpha=0.5, + ground_truth=False, + keyframe_idx=None, + draw_range=None, + repeat_frame=1, + ): + """ + Draw predicted labels or ground truth classes to clip. Draw bouding boxes to clip + if bboxes is provided. Boxes will gradually fade in and out the clip, centered around + the clip's central frame, within the provided `draw_range`. + Args: + frames (array-like): video data in the shape (T, H, W, C). + preds (tensor): a tensor of shape (num_boxes, num_classes) that contains all of the confidence scores + of the model. For recognition task or for ground_truth labels, input shape can be (num_classes,). + bboxes (Optional[tensor]): shape (num_boxes, 4) that contains the coordinates of the bounding boxes. + text_alpha (float): transparency label of the box wrapped around text labels. + ground_truth (bool): whether the prodived bounding boxes are ground-truth. + keyframe_idx (int): the index of keyframe in the clip. + draw_range (Optional[list[ints]): only draw frames in range [start_idx, end_idx] inclusively in the clip. + If None, draw on the entire clip. + repeat_frame (int): repeat each frame in draw_range for `repeat_frame` time for slow-motion effect. + """ + if draw_range is None: + draw_range = [0, len(frames) - 1] + if draw_range is not None: + draw_range[0] = max(0, draw_range[0]) + left_frames = frames[: draw_range[0]] + right_frames = frames[draw_range[1] + 1 :] + + draw_frames = frames[draw_range[0] : draw_range[1] + 1] + if keyframe_idx is None: + keyframe_idx = len(frames) // 2 + + img_ls = ( + list(left_frames) + + self.draw_clip( + draw_frames, + preds, + bboxes=bboxes, + text_alpha=text_alpha, + ground_truth=ground_truth, + keyframe_idx=keyframe_idx - draw_range[0], + repeat_frame=repeat_frame, + ) + + list(right_frames) + ) + + return img_ls + + def draw_clip( + self, + frames, + preds, + bboxes=None, + text_alpha=0.5, + ground_truth=False, + keyframe_idx=None, + repeat_frame=1, + ): + """ + Draw predicted labels or ground truth classes to clip. Draw bouding boxes to clip + if bboxes is provided. Boxes will gradually fade in and out the clip, centered around + the clip's central frame. + Args: + frames (array-like): video data in the shape (T, H, W, C). + preds (tensor): a tensor of shape (num_boxes, num_classes) that contains all of the confidence scores + of the model. For recognition task or for ground_truth labels, input shape can be (num_classes,). + bboxes (Optional[tensor]): shape (num_boxes, 4) that contains the coordinates of the bounding boxes. + text_alpha (float): transparency label of the box wrapped around text labels. + ground_truth (bool): whether the prodived bounding boxes are ground-truth. + keyframe_idx (int): the index of keyframe in the clip. + repeat_frame (int): repeat each frame in draw_range for `repeat_frame` time for slow-motion effect. + """ + assert repeat_frame >= 1, "`repeat_frame` must be a positive integer." + + repeated_seq = range(0, len(frames)) + repeated_seq = list( + itertools.chain.from_iterable( + itertools.repeat(x, repeat_frame) for x in repeated_seq + ) + ) + + frames, adjusted = self._adjust_frames_type(frames) + if keyframe_idx is None: + half_left = len(repeated_seq) // 2 + half_right = (len(repeated_seq) + 1) // 2 + else: + mid = int((keyframe_idx / len(frames)) * len(repeated_seq)) + half_left = mid + half_right = len(repeated_seq) - mid + + alpha_ls = np.concatenate( + [ + np.linspace(0, 1, num=half_left), + np.linspace(1, 0, num=half_right), + ] + ) + text_alpha = text_alpha + frames = frames[repeated_seq] + img_ls = [] + for alpha, frame in zip(alpha_ls, frames): + draw_img = self.draw_one_frame( + frame, + preds, + bboxes, + alpha=alpha, + text_alpha=text_alpha, + ground_truth=ground_truth, + ) + if adjusted: + draw_img = draw_img.astype("float32") / 255 + + img_ls.append(draw_img) + + return img_ls + + def _adjust_frames_type(self, frames): + """ + Modify video data to have dtype of uint8 and values range in [0, 255]. + Args: + frames (array-like): 4D array of shape (T, H, W, C). + Returns: + frames (list of frames): list of frames in range [0, 1]. + adjusted (bool): whether the original frames need adjusted. + """ + assert ( + frames is not None and len(frames) != 0 + ), "Frames does not contain any values" + frames = np.array(frames) + assert np.array(frames).ndim == 4, "Frames must have 4 dimensions" + adjusted = False + if frames.dtype in [np.float32, np.float64]: + frames *= 255 + frames = frames.astype(np.uint8) + adjusted = True + + return frames, adjusted + + def _get_thres_array(self, common_class_names=None): + """ + Compute a thresholds array for all classes based on `self.thes` and `self.lower_thres`. + Args: + common_class_names (Optional[list of strs]): a list of common class names. + """ + common_class_ids = [] + if common_class_names is not None: + common_classes = set(common_class_names) + + for i, name in enumerate(self.class_names): + if name in common_classes: + common_class_ids.append(i) + else: + common_class_ids = list(range(self.num_classes)) + + thres_array = np.full( + shape=(self.num_classes,), fill_value=self.lower_thres + ) + thres_array[common_class_ids] = self.thres + self.thres = torch.from_numpy(thres_array) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/benchmark.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/benchmark.py" new file mode 100644 index 0000000..acdc12a --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/benchmark.py" @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved +""" +A script to benchmark data loading. +""" + +import slowfast.utils.logging as logging +from slowfast.utils.benchmark import benchmark_data_loading +from slowfast.utils.misc import launch_job +from slowfast.utils.parser import load_config, parse_args + +logger = logging.get_logger(__name__) + + +def main(): + args = parse_args() + cfg = load_config(args) + + launch_job( + cfg=cfg, init_method=args.init_method, func=benchmark_data_loading + ) + + +if __name__ == "__main__": + main() diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/demo_net.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/demo_net.py" new file mode 100644 index 0000000..a7e98eb --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/demo_net.py" @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import numpy as np +import time +import torch +import tqdm + +from slowfast.utils import logging +from slowfast.visualization.async_predictor import AsyncDemo, AsyncVis +from slowfast.visualization.ava_demo_precomputed_boxes import ( + AVAVisualizerWithPrecomputedBox, +) +from slowfast.visualization.demo_loader import ThreadVideoManager, VideoManager +from slowfast.visualization.predictor import ActionPredictor +from slowfast.visualization.video_visualizer import VideoVisualizer + +logger = logging.get_logger(__name__) + + +def run_demo(cfg, frame_provider): + """ + Run demo visualization. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + frame_provider (iterator): Python iterator that return task objects that are filled + with necessary information such as `frames`, `id` and `num_buffer_frames` for the + prediction and visualization pipeline. + """ + # Set random seed from configs. + np.random.seed(cfg.RNG_SEED) + torch.manual_seed(cfg.RNG_SEED) + # Setup logging format. + logging.setup_logging(cfg.OUTPUT_DIR) + # Print config. + logger.info("Run demo with config:") + logger.info(cfg) + common_classes = ( + cfg.DEMO.COMMON_CLASS_NAMES + if len(cfg.DEMO.LABEL_FILE_PATH) != 0 + else None + ) + + video_vis = VideoVisualizer( + num_classes=cfg.MODEL.NUM_CLASSES, + class_names_path=cfg.DEMO.LABEL_FILE_PATH, + top_k=cfg.TENSORBOARD.MODEL_VIS.TOPK_PREDS, + thres=cfg.DEMO.COMMON_CLASS_THRES, + lower_thres=cfg.DEMO.UNCOMMON_CLASS_THRES, + common_class_names=common_classes, + colormap=cfg.TENSORBOARD.MODEL_VIS.COLORMAP, + mode=cfg.DEMO.VIS_MODE, + ) + + async_vis = AsyncVis(video_vis, n_workers=cfg.DEMO.NUM_VIS_INSTANCES) + + if cfg.NUM_GPUS <= 1: + model = ActionPredictor(cfg=cfg, async_vis=async_vis) + else: + model = AsyncDemo(cfg=cfg, async_vis=async_vis) + + seq_len = cfg.DATA.NUM_FRAMES * cfg.DATA.SAMPLING_RATE + + assert ( + cfg.DEMO.BUFFER_SIZE <= seq_len // 2 + ), "Buffer size cannot be greater than half of sequence length." + num_task = 0 + # Start reading frames. + frame_provider.start() + for able_to_read, task in frame_provider: + if not able_to_read: + break + if task is None: + time.sleep(0.02) + continue + num_task += 1 + + model.put(task) + try: + task = model.get() + num_task -= 1 + yield task + except IndexError: + continue + + while num_task != 0: + try: + task = model.get() + num_task -= 1 + yield task + except IndexError: + continue + + +def demo(cfg): + """ + Run inference on an input video or stream from webcam. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + # AVA format-specific visualization with precomputed boxes. + if cfg.DETECTION.ENABLE and cfg.DEMO.PREDS_BOXES != "": + precomputed_box_vis = AVAVisualizerWithPrecomputedBox(cfg) + precomputed_box_vis() + else: + start = time.time() + if cfg.DEMO.THREAD_ENABLE: + frame_provider = ThreadVideoManager(cfg) + else: + frame_provider = VideoManager(cfg) + + for task in tqdm.tqdm(run_demo(cfg, frame_provider)): + frame_provider.display(task) + + frame_provider.join() + frame_provider.clean() + logger.info("Finish demo in: {}".format(time.time() - start)) diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/run_net.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/run_net.py" new file mode 100644 index 0000000..c26e124 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/run_net.py" @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Wrapper to train and test a video classification model.""" +from slowfast.config.defaults import assert_and_infer_cfg +from slowfast.utils.misc import launch_job +from slowfast.utils.parser import load_config, parse_args + +from demo_net import demo +from test_net import test +from train_net import train +from visualization import visualize + +import os + +def main(): + test_path="./Vinput/action" + test_list=os.listdir(test_path) + for file in test_list: + + """ + Main function to spawn the train and test process. + """ + args = parse_args() + cfg = load_config(args) + cfg = assert_and_infer_cfg(cfg) + + # Perform training. + if cfg.TRAIN.ENABLE: + launch_job(cfg=cfg, init_method=args.init_method, func=train) + + # Perform multi-clip testing. + if cfg.TEST.ENABLE: + launch_job(cfg=cfg, init_method=args.init_method, func=test) + + # Perform model visualization. + if cfg.TENSORBOARD.ENABLE and ( + cfg.TENSORBOARD.MODEL_VIS.ENABLE + or cfg.TENSORBOARD.WRONG_PRED_VIS.ENABLE + ): + launch_job(cfg=cfg, init_method=args.init_method, func=visualize) + + # Run demo. + if cfg.DEMO.ENABLE: + demo(cfg) + + +if __name__ == "__main__": + main() diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/ss.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/ss.py" new file mode 100644 index 0000000..268c902 --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/ss.py" @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +import os +path = "./" +filelist = os.listdir(path) #该文件夹下所有的文件（包括文件夹） +count=1 +for file in filelist: + print(file) +for file in filelist: #遍历所有文件 + Olddir=os.path.join(path,file) #原来的文件路径 + if os.path.isdir(Olddir): #如果是文件夹则跳过 + continue + filename=os.path.splitext(file)[0] #文件名 + filetype=os.path.splitext(file)[1] #文件扩展名 + filename="zxw" + Newdir=os.path.join(path,filename+str(count).zfill(4)+filetype) #用字符串函数zfill 以0补全所需位数 + if(len(Olddir)>8): + count1=count + os.rename(Olddir,Olddir[:5]+'_'+str("{:0>6d}".format(count1))+'.jpg')#重命名 + count+=1 \ No newline at end of file diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/test_net.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/test_net.py" new file mode 100644 index 0000000..550b1bf --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/test_net.py" @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Multi-view test a video classification model.""" + +import numpy as np +import os +import pickle +import torch +from iopath.common.file_io import g_pathmgr + +import slowfast.utils.checkpoint as cu +import slowfast.utils.distributed as du +import slowfast.utils.logging as logging +import slowfast.utils.misc as misc +import slowfast.visualization.tensorboard_vis as tb +from slowfast.datasets import loader +from slowfast.models import build_model +from slowfast.utils.meters import AVAMeter, TestMeter + +logger = logging.get_logger(__name__) + + +@torch.no_grad() +def perform_test(test_loader, model, test_meter, cfg, writer=None): + """ + For classification: + Perform mutli-view testing that uniformly samples N clips from a video along + its temporal axis. For each clip, it takes 3 crops to cover the spatial + dimension, followed by averaging the softmax scores across all Nx3 views to + form a video-level prediction. All video predictions are compared to + ground-truth labels and the final testing performance is logged. + For detection: + Perform fully-convolutional testing on the full frames without crop. + Args: + test_loader (loader): video testing loader. + model (model): the pretrained video model to test. + test_meter (TestMeter): testing meters to log and ensemble the testing + results. + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + writer (TensorboardWriter object, optional): TensorboardWriter object + to writer Tensorboard log. + """ + # Enable eval mode. + model.eval() + test_meter.iter_tic() + + for cur_iter, (inputs, labels, video_idx, meta) in enumerate(test_loader): + if cfg.NUM_GPUS: + # Transfer the data to the current GPU device. + if isinstance(inputs, (list,)): + for i in range(len(inputs)): + inputs[i] = inputs[i].cuda(non_blocking=True) + else: + inputs = inputs.cuda(non_blocking=True) + + # Transfer the data to the current GPU device. + labels = labels.cuda() + video_idx = video_idx.cuda() + for key, val in meta.items(): + if isinstance(val, (list,)): + for i in range(len(val)): + val[i] = val[i].cuda(non_blocking=True) + else: + meta[key] = val.cuda(non_blocking=True) + test_meter.data_toc() + + if cfg.DETECTION.ENABLE: + # Compute the predictions. + preds = model(inputs, meta["boxes"]) + ori_boxes = meta["ori_boxes"] + metadata = meta["metadata"] + + preds = preds.detach().cpu() if cfg.NUM_GPUS else preds.detach() + ori_boxes = ( + ori_boxes.detach().cpu() if cfg.NUM_GPUS else ori_boxes.detach() + ) + metadata = ( + metadata.detach().cpu() if cfg.NUM_GPUS else metadata.detach() + ) + + if cfg.NUM_GPUS > 1: + preds = torch.cat(du.all_gather_unaligned(preds), dim=0) + ori_boxes = torch.cat(du.all_gather_unaligned(ori_boxes), dim=0) + metadata = torch.cat(du.all_gather_unaligned(metadata), dim=0) + + test_meter.iter_toc() + # Update and log stats. + test_meter.update_stats(preds, ori_boxes, metadata) + test_meter.log_iter_stats(None, cur_iter) + else: + # Perform the forward pass. + preds = model(inputs) + + # Gather all the predictions across all the devices to perform ensemble. + if cfg.NUM_GPUS > 1: + preds, labels, video_idx = du.all_gather( + [preds, labels, video_idx] + ) + if cfg.NUM_GPUS: + preds = preds.cpu() + labels = labels.cpu() + video_idx = video_idx.cpu() + + test_meter.iter_toc() + # Update and log stats. + test_meter.update_stats( + preds.detach(), labels.detach(), video_idx.detach() + ) + test_meter.log_iter_stats(cur_iter) + + test_meter.iter_tic() + + # Log epoch stats and print the final testing results. + if not cfg.DETECTION.ENABLE: + all_preds = test_meter.video_preds.clone().detach() + all_labels = test_meter.video_labels + if cfg.NUM_GPUS: + all_preds = all_preds.cpu() + all_labels = all_labels.cpu() + if writer is not None: + writer.plot_eval(preds=all_preds, labels=all_labels) + + if cfg.TEST.SAVE_RESULTS_PATH != "": + save_path = os.path.join(cfg.OUTPUT_DIR, cfg.TEST.SAVE_RESULTS_PATH) + + if du.is_root_proc(): + with g_pathmgr.open(save_path, "wb") as f: + pickle.dump([all_preds, all_labels], f) + + logger.info( + "Successfully saved prediction results to {}".format(save_path) + ) + + test_meter.finalize_metrics() + return test_meter + + +def test(cfg): + """ + Perform multi-view testing on the pretrained video model. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + # Set up environment. + du.init_distributed_training(cfg) + # Set random seed from configs. + np.random.seed(cfg.RNG_SEED) + torch.manual_seed(cfg.RNG_SEED) + + # Setup logging format. + logging.setup_logging(cfg.OUTPUT_DIR) + + # Print config. + logger.info("Test with config:") + logger.info(cfg) + + # Build the video model and print model statistics. + model = build_model(cfg) + if du.is_master_proc() and cfg.LOG_MODEL_INFO: + misc.log_model_info(model, cfg, use_train_input=False) + + cu.load_test_checkpoint(cfg, model) + + # Create video testing loaders. + test_loader = loader.construct_loader(cfg, "test") + logger.info("Testing model for {} iterations".format(len(test_loader))) + + if cfg.DETECTION.ENABLE: + assert cfg.NUM_GPUS == cfg.TEST.BATCH_SIZE or cfg.NUM_GPUS == 0 + test_meter = AVAMeter(len(test_loader), cfg, mode="test") + else: + assert ( + test_loader.dataset.num_videos + % (cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS) + == 0 + ) + # Create meters for multi-view testing. + test_meter = TestMeter( + test_loader.dataset.num_videos + // (cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS), + cfg.TEST.NUM_ENSEMBLE_VIEWS * cfg.TEST.NUM_SPATIAL_CROPS, + cfg.MODEL.NUM_CLASSES, + len(test_loader), + cfg.DATA.MULTI_LABEL, + cfg.DATA.ENSEMBLE_METHOD, + ) + + # Set up writer for logging to Tensorboard format. + if cfg.TENSORBOARD.ENABLE and du.is_master_proc( + cfg.NUM_GPUS * cfg.NUM_SHARDS + ): + writer = tb.TensorboardWriter(cfg) + else: + writer = None + + # # Perform multi-view test on the entire dataset. + test_meter = perform_test(test_loader, model, test_meter, cfg, writer) + if writer is not None: + writer.close() diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/train_net.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/train_net.py" new file mode 100644 index 0000000..24a861c --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/train_net.py" @@ -0,0 +1,524 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +"""Train a video classification model.""" + +import numpy as np +import pprint +import torch +from fvcore.nn.precise_bn import get_bn_modules, update_bn_stats + +import slowfast.models.losses as losses +import slowfast.models.optimizer as optim +import slowfast.utils.checkpoint as cu +import slowfast.utils.distributed as du +import slowfast.utils.logging as logging +import slowfast.utils.metrics as metrics +import slowfast.utils.misc as misc +import slowfast.visualization.tensorboard_vis as tb +from slowfast.datasets import loader +from slowfast.datasets.mixup import MixUp +from slowfast.models import build_model +from slowfast.utils.meters import AVAMeter, EpochTimer, TrainMeter, ValMeter +from slowfast.utils.multigrid import MultigridSchedule + +logger = logging.get_logger(__name__) + + +def train_epoch( + train_loader, model, optimizer, train_meter, cur_epoch, cfg, writer=None +): + """ + Perform the video training for one epoch. + Args: + train_loader (loader): video training loader. + model (model): the video model to train. + optimizer (optim): the optimizer to perform optimization on the model's + parameters. + train_meter (TrainMeter): training meters to log the training performance. + cur_epoch (int): current epoch of training. + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + writer (TensorboardWriter, optional): TensorboardWriter object + to writer Tensorboard log. + """ + # Enable train mode. + model.train() + train_meter.iter_tic() + data_size = len(train_loader) + + if cfg.MIXUP.ENABLE: + mixup_fn = MixUp( + mixup_alpha=cfg.MIXUP.ALPHA, + cutmix_alpha=cfg.MIXUP.CUTMIX_ALPHA, + mix_prob=cfg.MIXUP.PROB, + switch_prob=cfg.MIXUP.SWITCH_PROB, + label_smoothing=cfg.MIXUP.LABEL_SMOOTH_VALUE, + num_classes=cfg.MODEL.NUM_CLASSES, + ) + + for cur_iter, (inputs, labels, _, meta) in enumerate(train_loader): + # Transfer the data to the current GPU device. + if cfg.NUM_GPUS: + if isinstance(inputs, (list,)): + for i in range(len(inputs)): + inputs[i] = inputs[i].cuda(non_blocking=True) + else: + inputs = inputs.cuda(non_blocking=True) + labels = labels.cuda() + for key, val in meta.items(): + if isinstance(val, (list,)): + for i in range(len(val)): + val[i] = val[i].cuda(non_blocking=True) + else: + meta[key] = val.cuda(non_blocking=True) + + # Update the learning rate. + lr = optim.get_epoch_lr(cur_epoch + float(cur_iter) / data_size, cfg) + optim.set_lr(optimizer, lr) + + train_meter.data_toc() + if cfg.MIXUP.ENABLE: + samples, labels = mixup_fn(inputs[0], labels) + inputs[0] = samples + + if cfg.DETECTION.ENABLE: + preds = model(inputs, meta["boxes"]) + else: + preds = model(inputs) + # Explicitly declare reduction to mean. + loss_fun = losses.get_loss_func(cfg.MODEL.LOSS_FUNC)(reduction="mean") + + # Compute the loss. + loss = loss_fun(preds, labels) + + # check Nan Loss. + misc.check_nan_losses(loss) + + # Perform the backward pass. + optimizer.zero_grad() + loss.backward() + # Update the parameters. + optimizer.step() + + if cfg.MIXUP.ENABLE: + _top_max_k_vals, top_max_k_inds = torch.topk( + labels, 2, dim=1, largest=True, sorted=True + ) + idx_top1 = torch.arange(labels.shape[0]), top_max_k_inds[:, 0] + idx_top2 = torch.arange(labels.shape[0]), top_max_k_inds[:, 1] + preds = preds.detach() + preds[idx_top1] += preds[idx_top2] + preds[idx_top2] = 0.0 + labels = top_max_k_inds[:, 0] + + if cfg.DETECTION.ENABLE: + if cfg.NUM_GPUS > 1: + loss = du.all_reduce([loss])[0] + loss = loss.item() + + # Update and log stats. + train_meter.update_stats(None, None, None, loss, lr) + # write to tensorboard format if available. + if writer is not None: + writer.add_scalars( + {"Train/loss": loss, "Train/lr": lr}, + global_step=data_size * cur_epoch + cur_iter, + ) + + else: + top1_err, top5_err = None, None + if cfg.DATA.MULTI_LABEL: + # Gather all the predictions across all the devices. + if cfg.NUM_GPUS > 1: + [loss] = du.all_reduce([loss]) + loss = loss.item() + else: + # Compute the errors. + num_topks_correct = metrics.topks_correct(preds, labels, (1, 5)) + top1_err, top5_err = [ + (1.0 - x / preds.size(0)) * 100.0 for x in num_topks_correct + ] + # Gather all the predictions across all the devices. + if cfg.NUM_GPUS > 1: + loss, top1_err, top5_err = du.all_reduce( + [loss, top1_err, top5_err] + ) + + # Copy the stats from GPU to CPU (sync point). + loss, top1_err, top5_err = ( + loss.item(), + top1_err.item(), + top5_err.item(), + ) + + # Update and log stats. + train_meter.update_stats( + top1_err, + top5_err, + loss, + lr, + inputs[0].size(0) + * max( + cfg.NUM_GPUS, 1 + ), # If running on CPU (cfg.NUM_GPUS == 1), use 1 to represent 1 CPU. + ) + # write to tensorboard format if available. + if writer is not None: + writer.add_scalars( + { + "Train/loss": loss, + "Train/lr": lr, + "Train/Top1_err": top1_err, + "Train/Top5_err": top5_err, + }, + global_step=data_size * cur_epoch + cur_iter, + ) + + train_meter.iter_toc() # measure allreduce for this meter + train_meter.log_iter_stats(cur_epoch, cur_iter) + train_meter.iter_tic() + + # Log epoch stats. + train_meter.log_epoch_stats(cur_epoch) + train_meter.reset() + + +@torch.no_grad() +def eval_epoch(val_loader, model, val_meter, cur_epoch, cfg, writer=None): + """ + Evaluate the model on the val set. + Args: + val_loader (loader): data loader to provide validation data. + model (model): model to evaluate the performance. + val_meter (ValMeter): meter instance to record and calculate the metrics. + cur_epoch (int): number of the current epoch of training. + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + writer (TensorboardWriter, optional): TensorboardWriter object + to writer Tensorboard log. + """ + + # Evaluation mode enabled. The running stats would not be updated. + model.eval() + val_meter.iter_tic() + + for cur_iter, (inputs, labels, _, meta) in enumerate(val_loader): + if cfg.NUM_GPUS: + # Transferthe data to the current GPU device. + if isinstance(inputs, (list,)): + for i in range(len(inputs)): + inputs[i] = inputs[i].cuda(non_blocking=True) + else: + inputs = inputs.cuda(non_blocking=True) + labels = labels.cuda() + for key, val in meta.items(): + if isinstance(val, (list,)): + for i in range(len(val)): + val[i] = val[i].cuda(non_blocking=True) + else: + meta[key] = val.cuda(non_blocking=True) + val_meter.data_toc() + + if cfg.DETECTION.ENABLE: + # Compute the predictions. + preds = model(inputs, meta["boxes"]) + ori_boxes = meta["ori_boxes"] + metadata = meta["metadata"] + + if cfg.NUM_GPUS: + preds = preds.cpu() + ori_boxes = ori_boxes.cpu() + metadata = metadata.cpu() + + if cfg.NUM_GPUS > 1: + preds = torch.cat(du.all_gather_unaligned(preds), dim=0) + ori_boxes = torch.cat(du.all_gather_unaligned(ori_boxes), dim=0) + metadata = torch.cat(du.all_gather_unaligned(metadata), dim=0) + + val_meter.iter_toc() + # Update and log stats. + val_meter.update_stats(preds, ori_boxes, metadata) + + else: + preds = model(inputs) + + if cfg.DATA.MULTI_LABEL: + if cfg.NUM_GPUS > 1: + preds, labels = du.all_gather([preds, labels]) + else: + # Compute the errors. + num_topks_correct = metrics.topks_correct(preds, labels, (1, 5)) + + # Combine the errors across the GPUs. + top1_err, top5_err = [ + (1.0 - x / preds.size(0)) * 100.0 for x in num_topks_correct + ] + if cfg.NUM_GPUS > 1: + top1_err, top5_err = du.all_reduce([top1_err, top5_err]) + + # Copy the errors from GPU to CPU (sync point). + top1_err, top5_err = top1_err.item(), top5_err.item() + + val_meter.iter_toc() + # Update and log stats. + val_meter.update_stats( + top1_err, + top5_err, + inputs[0].size(0) + * max( + cfg.NUM_GPUS, 1 + ), # If running on CPU (cfg.NUM_GPUS == 1), use 1 to represent 1 CPU. + ) + # write to tensorboard format if available. + if writer is not None: + writer.add_scalars( + {"Val/Top1_err": top1_err, "Val/Top5_err": top5_err}, + global_step=len(val_loader) * cur_epoch + cur_iter, + ) + + val_meter.update_predictions(preds, labels) + + val_meter.log_iter_stats(cur_epoch, cur_iter) + val_meter.iter_tic() + + # Log epoch stats. + val_meter.log_epoch_stats(cur_epoch) + # write to tensorboard format if available. + if writer is not None: + if cfg.DETECTION.ENABLE: + writer.add_scalars( + {"Val/mAP": val_meter.full_map}, global_step=cur_epoch + ) + else: + all_preds = [pred.clone().detach() for pred in val_meter.all_preds] + all_labels = [ + label.clone().detach() for label in val_meter.all_labels + ] + if cfg.NUM_GPUS: + all_preds = [pred.cpu() for pred in all_preds] + all_labels = [label.cpu() for label in all_labels] + writer.plot_eval( + preds=all_preds, labels=all_labels, global_step=cur_epoch + ) + + val_meter.reset() + + +def calculate_and_update_precise_bn(loader, model, num_iters=200, use_gpu=True): + """ + Update the stats in bn layers by calculate the precise stats. + Args: + loader (loader): data loader to provide training data. + model (model): model to update the bn stats. + num_iters (int): number of iterations to compute and update the bn stats. + use_gpu (bool): whether to use GPU or not. + """ + + def _gen_loader(): + for inputs, *_ in loader: + if use_gpu: + if isinstance(inputs, (list,)): + for i in range(len(inputs)): + inputs[i] = inputs[i].cuda(non_blocking=True) + else: + inputs = inputs.cuda(non_blocking=True) + yield inputs + + # Update the bn stats. + update_bn_stats(model, _gen_loader(), num_iters) + + +def build_trainer(cfg): + """ + Build training model and its associated tools, including optimizer, + dataloaders and meters. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + Returns: + model (nn.Module): training model. + optimizer (Optimizer): optimizer. + train_loader (DataLoader): training data loader. + val_loader (DataLoader): validatoin data loader. + precise_bn_loader (DataLoader): training data loader for computing + precise BN. + train_meter (TrainMeter): tool for measuring training stats. + val_meter (ValMeter): tool for measuring validation stats. + """ + # Build the video model and print model statistics. + model = build_model(cfg) + if du.is_master_proc() and cfg.LOG_MODEL_INFO: + misc.log_model_info(model, cfg, use_train_input=True) + + # Construct the optimizer. + optimizer = optim.construct_optimizer(model, cfg) + + # Create the video train and val loaders. + train_loader = loader.construct_loader(cfg, "train") + val_loader = loader.construct_loader(cfg, "val") + precise_bn_loader = loader.construct_loader( + cfg, "train", is_precise_bn=True + ) + # Create meters. + train_meter = TrainMeter(len(train_loader), cfg) + val_meter = ValMeter(len(val_loader), cfg) + + return ( + model, + optimizer, + train_loader, + val_loader, + precise_bn_loader, + train_meter, + val_meter, + ) + + +def train(cfg): + """ + Train a video model for many epochs on train set and evaluate it on val set. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + # Set up environment. + du.init_distributed_training(cfg) + # Set random seed from configs. + np.random.seed(cfg.RNG_SEED) + torch.manual_seed(cfg.RNG_SEED) + + # Setup logging format. + logging.setup_logging(cfg.OUTPUT_DIR) + + # Init multigrid. + multigrid = None + if cfg.MULTIGRID.LONG_CYCLE or cfg.MULTIGRID.SHORT_CYCLE: + multigrid = MultigridSchedule() + cfg = multigrid.init_multigrid(cfg) + if cfg.MULTIGRID.LONG_CYCLE: + cfg, _ = multigrid.update_long_cycle(cfg, cur_epoch=0) + # Print config. + logger.info("Train with config:") + logger.info(pprint.pformat(cfg)) + + # Build the video model and print model statistics. + model = build_model(cfg) + if du.is_master_proc() and cfg.LOG_MODEL_INFO: + misc.log_model_info(model, cfg, use_train_input=True) + + # Construct the optimizer. + optimizer = optim.construct_optimizer(model, cfg) + + # Load a checkpoint to resume training if applicable. + start_epoch = cu.load_train_checkpoint(cfg, model, optimizer) + + # Create the video train and val loaders. + train_loader = loader.construct_loader(cfg, "train") + val_loader = loader.construct_loader(cfg, "val") + precise_bn_loader = ( + loader.construct_loader(cfg, "train", is_precise_bn=True) + if cfg.BN.USE_PRECISE_STATS + else None + ) + + # Create meters. + if cfg.DETECTION.ENABLE: + train_meter = AVAMeter(len(train_loader), cfg, mode="train") + val_meter = AVAMeter(len(val_loader), cfg, mode="val") + else: + train_meter = TrainMeter(len(train_loader), cfg) + val_meter = ValMeter(len(val_loader), cfg) + + # set up writer for logging to Tensorboard format. + if cfg.TENSORBOARD.ENABLE and du.is_master_proc( + cfg.NUM_GPUS * cfg.NUM_SHARDS + ): + writer = tb.TensorboardWriter(cfg) + else: + writer = None + + # Perform the training loop. + logger.info("Start epoch: {}".format(start_epoch + 1)) + + epoch_timer = EpochTimer() + + for cur_epoch in range(start_epoch, cfg.SOLVER.MAX_EPOCH): + if cfg.MULTIGRID.LONG_CYCLE: + cfg, changed = multigrid.update_long_cycle(cfg, cur_epoch) + if changed: + ( + model, + optimizer, + train_loader, + val_loader, + precise_bn_loader, + train_meter, + val_meter, + ) = build_trainer(cfg) + + # Load checkpoint. + if cu.has_checkpoint(cfg.OUTPUT_DIR): + last_checkpoint = cu.get_last_checkpoint(cfg.OUTPUT_DIR) + assert "{:05d}.pyth".format(cur_epoch) in last_checkpoint + else: + last_checkpoint = cfg.TRAIN.CHECKPOINT_FILE_PATH + logger.info("Load from {}".format(last_checkpoint)) + cu.load_checkpoint( + last_checkpoint, model, cfg.NUM_GPUS > 1, optimizer + ) + + # Shuffle the dataset. + loader.shuffle_dataset(train_loader, cur_epoch) + + # Train for one epoch. + epoch_timer.epoch_tic() + train_epoch( + train_loader, model, optimizer, train_meter, cur_epoch, cfg, writer + ) + epoch_timer.epoch_toc() + logger.info( + f"Epoch {cur_epoch} takes {epoch_timer.last_epoch_time():.2f}s. Epochs " + f"from {start_epoch} to {cur_epoch} take " + f"{epoch_timer.avg_epoch_time():.2f}s in average and " + f"{epoch_timer.median_epoch_time():.2f}s in median." + ) + logger.info( + f"For epoch {cur_epoch}, each iteraction takes " + f"{epoch_timer.last_epoch_time()/len(train_loader):.2f}s in average. " + f"From epoch {start_epoch} to {cur_epoch}, each iteraction takes " + f"{epoch_timer.avg_epoch_time()/len(train_loader):.2f}s in average." + ) + + is_checkp_epoch = cu.is_checkpoint_epoch( + cfg, + cur_epoch, + None if multigrid is None else multigrid.schedule, + ) + is_eval_epoch = misc.is_eval_epoch( + cfg, cur_epoch, None if multigrid is None else multigrid.schedule + ) + + # Compute precise BN stats. + if ( + (is_checkp_epoch or is_eval_epoch) + and cfg.BN.USE_PRECISE_STATS + and len(get_bn_modules(model)) > 0 + ): + calculate_and_update_precise_bn( + precise_bn_loader, + model, + min(cfg.BN.NUM_BATCHES_PRECISE, len(precise_bn_loader)), + cfg.NUM_GPUS > 0, + ) + _ = misc.aggregate_sub_bn_stats(model) + + # Save a checkpoint. + if is_checkp_epoch: + cu.save_checkpoint(cfg.OUTPUT_DIR, model, optimizer, cur_epoch, cfg) + # Evaluate the model on validation set. + if is_eval_epoch: + eval_epoch(val_loader, model, val_meter, cur_epoch, cfg, writer) + + if writer is not None: + writer.close() diff --git "a/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/visualization.py" "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/visualization.py" new file mode 100644 index 0000000..905863b --- /dev/null +++ "b/code/2021_spring/2019302110407-\345\274\240\347\277\224\345\256\207-\345\212\250\344\275\234/tools/visualization.py" @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + +import numpy as np +import pickle +import torch +import tqdm +from iopath.common.file_io import g_pathmgr + +import slowfast.datasets.utils as data_utils +import slowfast.utils.checkpoint as cu +import slowfast.utils.distributed as du +import slowfast.utils.logging as logging +import slowfast.utils.misc as misc +import slowfast.visualization.tensorboard_vis as tb +from slowfast.datasets import loader +from slowfast.models import build_model +from slowfast.visualization.gradcam_utils import GradCAM +from slowfast.visualization.prediction_vis import WrongPredictionVis +from slowfast.visualization.utils import ( + GetWeightAndActivation, + process_layer_index_data, +) +from slowfast.visualization.video_visualizer import VideoVisualizer + +logger = logging.get_logger(__name__) + + +def run_visualization(vis_loader, model, cfg, writer=None): + """ + Run model visualization (weights, activations and model inputs) and visualize + them on Tensorboard. + Args: + vis_loader (loader): video visualization loader. + model (model): the video model to visualize. + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + writer (TensorboardWriter, optional): TensorboardWriter object + to writer Tensorboard log. + """ + n_devices = cfg.NUM_GPUS * cfg.NUM_SHARDS + prefix = "module/" if n_devices > 1 else "" + # Get a list of selected layer names and indexing. + layer_ls, indexing_dict = process_layer_index_data( + cfg.TENSORBOARD.MODEL_VIS.LAYER_LIST, layer_name_prefix=prefix + ) + logger.info("Start Model Visualization.") + # Register hooks for activations. + model_vis = GetWeightAndActivation(model, layer_ls) + + if writer is not None and cfg.TENSORBOARD.MODEL_VIS.MODEL_WEIGHTS: + layer_weights = model_vis.get_weights() + writer.plot_weights_and_activations( + layer_weights, tag="Layer Weights/", heat_map=False + ) + + video_vis = VideoVisualizer( + cfg.MODEL.NUM_CLASSES, + cfg.TENSORBOARD.CLASS_NAMES_PATH, + cfg.TENSORBOARD.MODEL_VIS.TOPK_PREDS, + cfg.TENSORBOARD.MODEL_VIS.COLORMAP, + ) + if n_devices > 1: + grad_cam_layer_ls = [ + "module/" + layer + for layer in cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST + ] + else: + grad_cam_layer_ls = cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST + + if cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE: + gradcam = GradCAM( + model, + target_layers=grad_cam_layer_ls, + data_mean=cfg.DATA.MEAN, + data_std=cfg.DATA.STD, + colormap=cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.COLORMAP, + ) + logger.info("Finish drawing weights.") + global_idx = -1 + for inputs, labels, _, meta in tqdm.tqdm(vis_loader): + if cfg.NUM_GPUS: + # Transfer the data to the current GPU device. + if isinstance(inputs, (list,)): + for i in range(len(inputs)): + inputs[i] = inputs[i].cuda(non_blocking=True) + else: + inputs = inputs.cuda(non_blocking=True) + labels = labels.cuda() + for key, val in meta.items(): + if isinstance(val, (list,)): + for i in range(len(val)): + val[i] = val[i].cuda(non_blocking=True) + else: + meta[key] = val.cuda(non_blocking=True) + + if cfg.DETECTION.ENABLE: + activations, preds = model_vis.get_activations( + inputs, meta["boxes"] + ) + else: + activations, preds = model_vis.get_activations(inputs) + if cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE: + if cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.USE_TRUE_LABEL: + inputs, preds = gradcam(inputs, labels=labels) + else: + inputs, preds = gradcam(inputs) + if cfg.NUM_GPUS: + inputs = du.all_gather_unaligned(inputs) + activations = du.all_gather_unaligned(activations) + preds = du.all_gather_unaligned(preds) + if isinstance(inputs[0], list): + for i in range(len(inputs)): + for j in range(len(inputs[0])): + inputs[i][j] = inputs[i][j].cpu() + else: + inputs = [inp.cpu() for inp in inputs] + preds = [pred.cpu() for pred in preds] + else: + inputs, activations, preds = [inputs], [activations], [preds] + + boxes = [None] * max(n_devices, 1) + if cfg.DETECTION.ENABLE and cfg.NUM_GPUS: + boxes = du.all_gather_unaligned(meta["boxes"]) + boxes = [box.cpu() for box in boxes] + + if writer is not None: + total_vids = 0 + for i in range(max(n_devices, 1)): + cur_input = inputs[i] + cur_activations = activations[i] + cur_batch_size = cur_input[0].shape[0] + cur_preds = preds[i] + cur_boxes = boxes[i] + for cur_batch_idx in range(cur_batch_size): + global_idx += 1 + total_vids += 1 + if ( + cfg.TENSORBOARD.MODEL_VIS.INPUT_VIDEO + or cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE + ): + for path_idx, input_pathway in enumerate(cur_input): + if cfg.TEST.DATASET == "ava" and cfg.AVA.BGR: + video = input_pathway[ + cur_batch_idx, [2, 1, 0], ... + ] + else: + video = input_pathway[cur_batch_idx] + + if not cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE: + # Permute to (T, H, W, C) from (C, T, H, W). + video = video.permute(1, 2, 3, 0) + video = data_utils.revert_tensor_normalize( + video, cfg.DATA.MEAN, cfg.DATA.STD + ) + else: + # Permute from (T, C, H, W) to (T, H, W, C) + video = video.permute(0, 2, 3, 1) + bboxes = ( + None if cur_boxes is None else cur_boxes[:, 1:] + ) + cur_prediction = ( + cur_preds + if cfg.DETECTION.ENABLE + else cur_preds[cur_batch_idx] + ) + video = video_vis.draw_clip( + video, cur_prediction, bboxes=bboxes + ) + video = ( + torch.from_numpy(np.array(video)) + .permute(0, 3, 1, 2) + .unsqueeze(0) + ) + writer.add_video( + video, + tag="Input {}/Pathway {}".format( + global_idx, path_idx + 1 + ), + ) + if cfg.TENSORBOARD.MODEL_VIS.ACTIVATIONS: + writer.plot_weights_and_activations( + cur_activations, + tag="Input {}/Activations: ".format(global_idx), + batch_idx=cur_batch_idx, + indexing_dict=indexing_dict, + ) + + +def perform_wrong_prediction_vis(vis_loader, model, cfg): + """ + Visualize video inputs with wrong predictions on Tensorboard. + Args: + vis_loader (loader): video visualization loader. + model (model): the video model to visualize. + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + wrong_prediction_visualizer = WrongPredictionVis(cfg=cfg) + for batch_idx, (inputs, labels, _, _) in tqdm.tqdm(enumerate(vis_loader)): + if cfg.NUM_GPUS: + # Transfer the data to the current GPU device. + if isinstance(inputs, (list,)): + for i in range(len(inputs)): + inputs[i] = inputs[i].cuda(non_blocking=True) + else: + inputs = inputs.cuda(non_blocking=True) + labels = labels.cuda() + + # Some model modify the original input. + inputs_clone = [inp.clone() for inp in inputs] + + preds = model(inputs) + + if cfg.NUM_GPUS > 1: + preds, labels = du.all_gather([preds, labels]) + if isinstance(inputs_clone, (list,)): + inputs_clone = du.all_gather(inputs_clone) + else: + inputs_clone = du.all_gather([inputs_clone])[0] + + if cfg.NUM_GPUS: + # Transfer the data to the current CPU device. + labels = labels.cpu() + preds = preds.cpu() + if isinstance(inputs_clone, (list,)): + for i in range(len(inputs_clone)): + inputs_clone[i] = inputs_clone[i].cpu() + else: + inputs_clone = inputs_clone.cpu() + + # If using CPU (NUM_GPUS = 0), 1 represent 1 CPU. + n_devices = max(cfg.NUM_GPUS, 1) + for device_idx in range(1, n_devices + 1): + wrong_prediction_visualizer.visualize_vid( + video_input=inputs_clone, + labels=labels, + preds=preds.detach().clone(), + batch_idx=device_idx * batch_idx, + ) + + logger.info( + "Class indices with wrong predictions: {}".format( + sorted(wrong_prediction_visualizer.wrong_class_prediction) + ) + ) + wrong_prediction_visualizer.clean() + + +def visualize(cfg): + """ + Perform layer weights and activations visualization on the model. + Args: + cfg (CfgNode): configs. Details can be found in + slowfast/config/defaults.py + """ + if cfg.TENSORBOARD.ENABLE and ( + cfg.TENSORBOARD.MODEL_VIS.ENABLE + or cfg.TENSORBOARD.WRONG_PRED_VIS.ENABLE + ): + # Set up environment. + du.init_distributed_training(cfg) + # Set random seed from configs. + np.random.seed(cfg.RNG_SEED) + torch.manual_seed(cfg.RNG_SEED) + + # Setup logging format. + logging.setup_logging(cfg.OUTPUT_DIR) + + # Print config. + logger.info("Model Visualization with config:") + logger.info(cfg) + + # Build the video model and print model statistics. + model = build_model(cfg) + model.eval() + if du.is_master_proc() and cfg.LOG_MODEL_INFO: + misc.log_model_info(model, cfg, use_train_input=False) + + cu.load_test_checkpoint(cfg, model) + + # Create video testing loaders. + vis_loader = loader.construct_loader(cfg, "test") + + if cfg.DETECTION.ENABLE: + assert cfg.NUM_GPUS == cfg.TEST.BATCH_SIZE or cfg.NUM_GPUS == 0 + + # Set up writer for logging to Tensorboard format. + if du.is_master_proc(cfg.NUM_GPUS * cfg.NUM_SHARDS): + writer = tb.TensorboardWriter(cfg) + else: + writer = None + if cfg.TENSORBOARD.PREDICTIONS_PATH != "": + assert not cfg.DETECTION.ENABLE, "Detection is not supported." + logger.info( + "Visualizing class-level performance from saved results..." + ) + if writer is not None: + with g_pathmgr.open( + cfg.TENSORBOARD.PREDICTIONS_PATH, "rb" + ) as f: + preds, labels = pickle.load(f, encoding="latin1") + + writer.plot_eval(preds, labels) + + if cfg.TENSORBOARD.MODEL_VIS.ENABLE: + if cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.ENABLE: + assert ( + not cfg.DETECTION.ENABLE + ), "Detection task is currently not supported for Grad-CAM visualization." + if cfg.MODEL.ARCH in cfg.MODEL.SINGLE_PATHWAY_ARCH: + assert ( + len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST) == 1 + ), "The number of chosen CNN layers must be equal to the number of pathway(s), given {} layer(s).".format( + len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST) + ) + elif cfg.MODEL.ARCH in cfg.MODEL.MULTI_PATHWAY_ARCH: + assert ( + len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST) == 2 + ), "The number of chosen CNN layers must be equal to the number of pathway(s), given {} layer(s).".format( + len(cfg.TENSORBOARD.MODEL_VIS.GRAD_CAM.LAYER_LIST) + ) + else: + raise NotImplementedError( + "Model arch {} is not in {}".format( + cfg.MODEL.ARCH, + cfg.MODEL.SINGLE_PATHWAY_ARCH + + cfg.MODEL.MULTI_PATHWAY_ARCH, + ) + ) + logger.info( + "Visualize model analysis for {} iterations".format( + len(vis_loader) + ) + ) + # Run visualization on the model + run_visualization(vis_loader, model, cfg, writer) + if cfg.TENSORBOARD.WRONG_PRED_VIS.ENABLE: + logger.info( + "Visualize Wrong Predictions for {} iterations".format( + len(vis_loader) + ) + ) + perform_wrong_prediction_vis(vis_loader, model, cfg) + + if writer is not None: + writer.close() -- Gitee