3030
3131$$
3232\begin{aligned}
33- \frac{1}{n}\log\# &\le \frac{1}{n}\left(\log n\left(\frac{n}{e}\right)^n-\sum_{i=1}^M\log\left(\frac{np_i}{e}\right)^{np_i}\right)\\\\
34- &=\frac{1}{n}\log n-\sum_{i=1}^M\frac{np_i}{n}\log\frac{np_i}{n}\\\\
33+ \frac{1}{n}\log\# &\le \frac{1}{n}\left(\log n\left(\frac{n}{e}\right)^n-\sum_{i=1}^M\log\left(\frac{np_i}{e}\right)^{np_i}\right)\\
34+ &=\frac{1}{n}\log n-\sum_{i=1}^M\frac{np_i}{n}\log\frac{np_i}{n}\\
3535&\to -\sum_{i=1}^Mp_i\log p_i = H(X)\quad(n\to\infty)
3636\end{aligned}
3737$$
4040
4141$$
4242\begin{aligned}
43- \frac{1}{n}\log\# &\ge \frac{1}{n}\left(\log \left(\frac{n}{e}\right)^n-\sum_{i=1}^M\log\left(np_i\frac{np_i}{e}\right)^{np_i}\right)\\\\
44- &=\frac{1}{n}\log (n^Mp_1p_2\cdots p_M)-\sum_{i=1}^M\log\frac{np_i}{n}\\\\
43+ \frac{1}{n}\log\# &\ge \frac{1}{n}\left(\log \left(\frac{n}{e}\right)^n-\sum_{i=1}^M\log\left(np_i\frac{np_i}{e}\right)^{np_i}\right)\\
44+ &=\frac{1}{n}\log (n^Mp_1p_2\cdots p_M)-\sum_{i=1}^M\log\frac{np_i}{n}\\
4545&\to -\sum_{i=1}^Mp_i\log p_i = H(X)\quad(n\to\infty)
4646\end{aligned}
4747$$
7878
7979$$
8080\begin{aligned}
81- I(X;Y)&=H(X)+H(Y)-H(XY)\\\\
82- &=H(X)-H(X|Y)\\\\
81+ I(X;Y)&=H(X)+H(Y)-H(XY)\\
82+ &=H(X)-H(X|Y)\\
8383&=H(Y)-H(Y|X)
8484\end{aligned}
8585$$
9090
9191$$
9292 p_{i|j}=\begin{cases}
93- 1\,, x_i=f(\alpha_j)\\\\
93+ 1\,, x_i=f(\alpha_j)\\
9494 0\,, x_i\ne f(\alpha_j)
9595 \end{cases}
9696$$
121121
122122$$
123123\begin{aligned}
124- I(X;Y)&=h(X)+h(Y)-h(XY)\\\\
125- &=h(X)-h(Y|X)\\\\
124+ I(X;Y)&=h(X)+h(Y)-h(XY)\\
125+ &=h(X)-h(Y|X)\\
126126&=h(Y)-h(Y|X)
127127\end{aligned}
128128$$
@@ -134,6 +134,7 @@ p(x)=\frac{1}{2A}
134134$$
135135
136136给定** 方差约束** $\int_ {-\infty}^\infty p(x)x^2\mathrm dx=\sigma^2$\, 则最大微分熵分布为正态分布,熵为
137+
137138$$
138139h(X)=\frac{1}{2}\log2\pi\mathrm e\sigma^2
139140$$
184185
185186$$
186187\begin{aligned}
187- I(X;Y)&=H(Y)-H(Y|X)\\\\
188- &=H(Y)-\sum_ip_i\left(-\sum_jp_{j|i}\log p_{j|i}\right)\\\\
188+ I(X;Y)&=H(Y)-H(Y|X)\\
189+ &=H(Y)-\sum_ip_i\left(-\sum_jp_{j|i}\log p_{j|i}\right)\\
189190&=H(Y)-\left(-\varepsilon\log\varepsilon-(1-\varepsilon)\log(1-\varepsilon)\right)
190191\end{aligned}
191192$$
194195
195196$$
196197H(Y)\le 1\Leftrightarrow Y\sim\begin{pmatrix}
197- 0 & 1\\\\
198+ 0 & 1\\
1981991/2 & 1/2
199200\end{pmatrix}\Leftrightarrow X\sim\begin{pmatrix}
200- 0 & 1\\\\
201+ 0 & 1\\
2012021/2 & 1/2
202203\end{pmatrix}
203204$$
226227
227228$$
228229\begin{aligned}
229- I(X;Y)&=h(Y)-h(Y|X)\\\\
230- &=h(Y)-h(X+N|X)\\\\
230+ I(X;Y)&=h(Y)-h(Y|X)\\
231+ &=h(Y)-h(X+N|X)\\
231232&=h(Y)-h(N)
232233\end{aligned}
233234$$
236237
237238$$
238239\begin{aligned}
239- C&=\max_{p(x)}I(X;Y)\\\\
240- &=\max_{p(x)}h(X+N)-h(N)\\\\
241- &=\max_{p(x)}h(X+N)-\frac{1}{2}\log 2\pi\mathrm e\sigma^2\\\\
240+ C&=\max_{p(x)}I(X;Y)\\
241+ &=\max_{p(x)}h(X+N)-h(N)\\
242+ &=\max_{p(x)}h(X+N)-\frac{1}{2}\log 2\pi\mathrm e\sigma^2\\
242243\end{aligned}
243244$$
244245
258259
259260$$
260261\begin{aligned}
261- C&=\max_{p(x)}h(X+N)-\frac{1}{2}\log 2\pi\mathrm e \sigma^2\\\\
262- &=\frac{1}{2}\log 2\pi\mathrm e (P+\sigma^2)-\frac{1}{2}\log 2\pi\mathrm e \sigma^2\\\\
262+ C&=\max_{p(x)}h(X+N)-\frac{1}{2}\log 2\pi\mathrm e \sigma^2\\
263+ &=\frac{1}{2}\log 2\pi\mathrm e (P+\sigma^2)-\frac{1}{2}\log 2\pi\mathrm e \sigma^2\\
263264&=\boxed{\frac{1}{2}\log\left(1+\frac{P}{\sigma^2}\right)}
264265\end{aligned}
265266$$
270271
271272$$
272273\begin{aligned}
273- C&=\frac{1}{2}\log\left(1+\frac{P}{Wn_0}\right)\cdot2W\\\\
274+ C&=\frac{1}{2}\log\left(1+\frac{P}{Wn_0}\right)\cdot2W\\
274275&=\boxed{W\log\left(1+\frac{P}{Wn_0}\right)}
275276\end{aligned}
276277$$
369370
370371$$
371372\begin{aligned}
372- \sigma^2&=\int_{-\infty}^\infty [x-Q(x)]^2p(x)\mathrm{d}x\\\\
373+ \sigma^2&=\int_{-\infty}^\infty [x-Q(x)]^2p(x)\mathrm{d}x\\
373374&=\sum_{i=1}^L\int_{x_i}^{x_{i+1}}(x-y_i)^2p(x)\mathrm{d}x
374375\end{aligned}
375376$$
0 commit comments