$\newcommand{\bbx}[1]{\,\bbox[15px,border:1px groove navy]{\displaystyle{#1}}\,}
\newcommand{\braces}[1]{\left\lbrace\,{#1}\,\right\rbrace}
\newcommand{\bracks}[1]{\left\lbrack\,{#1}\,\right\rbrack}
\newcommand{\dd}{\mathrm{d}}
\newcommand{\ds}[1]{\displaystyle{#1}}
\newcommand{\expo}[1]{\,\mathrm{e}^{#1}\,}
\newcommand{\ic}{\mathrm{i}}
\newcommand{\mc}[1]{\mathcal{#1}}
\newcommand{\mrm}[1]{\mathrm{#1}}
\newcommand{\pars}[1]{\left(\,{#1}\,\right)}
\newcommand{\partiald}[3][]{\frac{\partial^{#1} #2}{\partial #3^{#1}}}
\newcommand{\root}[2][]{\,\sqrt[#1]{\,{#2}\,}\,}
\newcommand{\totald}[3][]{\frac{\mathrm{d}^{#1} #2}{\mathrm{d} #3^{#1}}}
\newcommand{\verts}[1]{\left\vert\,{#1}\,\right\vert}$
$\ds{\mbox{Integrand} = {x^{2}\sin\pars{x} \over 1 - 2a\cos\pars{x} + a^{2}}}$.
$$
\substack{Integrand\\ Behaviour}:\quad
\left\{\begin{array}{l}
\ds{\mrm{As}\ a \to -1\,,\ \sim {1 \over 2}\,{x^{2}\sin^{2}\pars{x} \over 1 + \cos\pars{x}} =
{1 \over 4}\,{x^{2}\sin^{2}\pars{x} \over \cos^{2}\pars{x/2}} =
x^{2}\sin^{2}\pars{x \over 2}}
\\[3mm]
\ds{\mrm{As}\ a \to \phantom{-\,}1\,,\ \sim {1 \over 2}\,{x^{2}\sin^{2}\pars{x} \over
1 - \cos\pars{x}} =
{1 \over 4}\,{x^{2}\sin^{2}\pars{x} \over \sin^{2}\pars{x/2}} =
x^{2}\cos^{2}\pars{x \over 2}}
\end{array}\right.
$$
- As $\ds{a \to -1}$, the integrand has an integrable singularity at
$\ds{x = \pi}$.
- As $\ds{a \to +1}$, the integrand has an integrable singularity at
$\ds{x = 0}$ and at $\ds{x = 2\pi}$.
With $\ds{a \in \mathbb{R}\setminus\braces{1}}$:
\begin{align}
&\int_{0}^{2\pi}{x^{2}\sin\pars{x} \over 1 - 2a\cos\pars{x} + a^{2}}
\,\dd x =
\int_{-\pi}^{\pi}{\pars{x^{2} + 2\pi x + \pi^{2}}\bracks{-\sin\pars{x}} \over
1 + 2a\cos\pars{x} + a^{2}}\,\dd x
\\[5mm] = &\
-4\pi\int_{0}^{\pi}{x\sin\pars{x} \over
1 + 2a\cos\pars{x} + a^{2}}\,\dd x
=
-4\pi\int_{0}^{\pi}{x\sin\pars{x} \over
\pars{a + \expo{\ic x}}\pars{a + \expo{-\ic x}}}\,\dd x
\\[5mm] = &\
-4\pi\int_{0}^{\pi}x\sin\pars{x}\,
\pars{{1 \over a + \expo{-\ic x}} - {1 \over a + \expo{\ic x}}}
\,{1 \over \expo{\ic x} - \expo{-\ic x}}\,\dd x
\\[5mm] = &\
4\pi\int_{0}^{\pi}x\sin\pars{x}
\pars{{1 \over a + \expo{\ic x}} - {1 \over a + \expo{-\ic x}}}
{1 \over 2\ic\sin\pars{x}}\,\dd x =
4\pi\,\Im\int_{0}^{\pi}{x \over a + \expo{\ic x}}\,\dd x
\label{1}\tag{1}
\end{align}
$\ds{\Large \verts{a} < 1}.$
\begin{align}
\Im\int_{0}^{\pi}{x \over a + \expo{\ic x}}\,\dd x & =
\Im\int_{0}^{\pi}{x\expo{-\ic x} \over 1 + a\expo{-\ic x}}\,\dd x =
\sum_{n = 0}^{\infty}\pars{-a}^{n}\
\overbrace{\Im\int_{0}^{\pi}x\expo{-\pars{n + 1}\ic x}\,\dd x}
^{\ds{{\pars{-1}^{n + 1} \over n + 1}}\,\pi}
\\[5mm] & =
-\pi\sum_{n = 0}^{\infty}{a^{n} \over n + 1} =
-\,{\pi \over a}\sum_{n = 1}^{\infty}{a^{n} \over n} =
\bbx{\pi\,{\ln\pars{1 - a} \over a}}\label{2}\tag{2}
\end{align}
$\ds{\Large\verts{a} > 1}.$
\begin{align}
\Im\int_{0}^{\pi}{x \over a + \expo{\ic x}}\,\dd x & =
{1 \over a}\,\Im\int_{0}^{\pi}{x \over 1 + \pars{1/a}\expo{\ic x}}\,\dd x =
{1 \over a}\,\Im\int_{0}^{\pi}
{x\expo{-\ic x} \over \pars{1/a} + \expo{-\ic x}}\,\dd x
\\[5mm] & =
-\,{1 \over a^{2}}\,\Im\int_{0}^{\pi}{x \over \pars{1/a} + \expo{-\ic x}}\,\dd x =
{1 \over a^{2}}\,\Im\int_{0}^{\pi}{x \over \pars{1/a} + \expo{\ic x}}\,\dd x
\\[5mm] & =
{1 \over a^{2}}\,\pi\,{\ln\pars{1 - 1/a} \over 1/a} =
\pi\,{\ln\pars{1 - 1/a} \over a}\label{3}\tag{3}
\end{align}
Here, I used, the previous result, \eqref{2}.
With \eqref{1}, \eqref{2} and \eqref{3}:
$$
\left.\int_{0}^{2\pi}{x^{2}\sin\pars{x} \over 1 - 2a\cos\pars{x} + a^{2}}
\,\dd x\,\right\vert_{\ a\ \in\ \mathbb{R}\setminus\braces{-1}} =
\left\{\begin{array}{lcl}
\ds{4\pi^{2}\,{\ln\pars{1 - a} \over a}} & \mbox{if} & \ds{\verts{a} < 1}
\\[2mm]
\ds{4\pi^{2}\,{\ln\pars{1 - 1/a} \over a}} & \mbox{if} & \ds{\verts{a} > 1}
\end{array}\right.
$$
The cases $\ds{a \to 0}$ and $\ds{a \to -1}$ are given by $\ds{-4\pi^{2}}$ and
$\ds{-4\pi^{2}\ln\pars{2}}$, respectively. The integral diverges logarithmically when $\ds{a \to 1}$ $\ds{\pars{~\mbox{as}\ 4\pi^{2}\ln\pars{1 - a}~}}$.
The following picture is a plot for $\ds{a \in \pars{-4,4}\setminus\braces{1}}$: