$\begin{array}{rcl} \int_0^{\frac{\pi}{2}}\sin(2x)\ln^2(1+2\tan^2x)dx & = & -\int_0^{\frac{\pi}{2}}\ln^2\left(\frac{\cos^2x+2\sin^2x}{\cos^2x}\right)d(\cos^2x) \\ \\ &\overset{y=\cos^2x} = & \int_0^1\ln^2\left(\frac{ 2-y}{y}\right)dy \\ \\ &\overset{z=\frac{2-y}{y}} = & 2\int_1^{\infty}\frac{\ln^2z}{(1+z)^2}dz \\ \\ & = & -2\int_1^{\infty}\ln^2zd\left(\frac{1}{1+z}\right) \\ \\ & \overset{\text{ Kısmi İntegrasyon}}= & 4\int_1^{\infty}\frac{\ln z}{z(1+z)}dz \\ \\ & \overset{z\rightarrow \frac{1}{z}}= & 4\int_0^1\frac{-\ln z}{\frac{1}{z}(1+\frac{1}{z})}\frac{dz}{z^2} \\ \\ & = & -4\int_0^1\frac{\ln z}{z+1}dz \\ \\ & = & -4\left(-\frac{\pi^2}{12}\right) \\ \\ & = & \frac{\pi^2}{3} \end{array}$