{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", "from ipywidgets import *\n", "import matplotlib.pyplot as plt\n", "from IPython.display import set_matplotlib_formats\n", "set_matplotlib_formats('svg')\n", "import numpy as np\n", "import scipy.stats as stats\n", "import matplotlib.patches as mpatches" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "def chi_sq(k=1):\n", " fig, axes = plt.subplots(figsize=(10,5))\n", " x = np.linspace(0, 30, 1000)\n", " chi2 = stats.chi2.pdf(x, k)\n", " plt.plot(x, chi2)\n", " plt.xlabel(\"x\")\n", " plt.ylabel(\"f(x)\")\n", " plt.title(r\"$\\chi^2($k=\"+str(k)+\"$)$\")\n", " #plt.xlim(0,80)\n", " plt.grid()\n", " \n", "def c_crit():\n", " k=5\n", " chi_sq(k-1)\n", " c = stats.chi2.ppf(0.95, k-1)\n", " x = np.linspace(c,30,100)\n", " y = stats.chi2.pdf(x, k-1)\n", " plt.fill_between(x, y, color=\"#ff4c4c\", alpha=0.5)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Analiza danych jakościowych" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Schemat postępowania" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "- Sformułowanie $H_0$ i $H_1$\n", "- Obliczenie (oczekiwanych) liczności zdarzeń przy założeniu prawdziwości $H_0$\n", "- Obliczenie wartości statystyki -- na podstawie różnicy między licznościami oczekiwanymi a zaobserwowanymi\n", "- Porównanie z wartością krytyczną/obliczenie p-wartości\n", "- Decyzja" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Rozkład $\\chi^2$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$Z_i \\sim N(0,1)$\n", "\n", "$$Q = \\sum_{i=1}^{k}Z_i^2$$\n", "\n", "$$Q \\sim \\chi^2(k)$$" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "1de0153ac2df4fc8a6c5a8d68e123922", "version_major": 2, "version_minor": 0 }, "text/plain": [ "interactive(children=(IntSlider(value=1, description='k', max=40, min=1), Output()), _dom_classes=('widget-int…" ] }, "metadata": {}, "output_type": "display_data" }, { "data": { "text/plain": [ "" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "interact(chi_sq, k=(1,40,1))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Test zgodności $\\chi^2$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "- k: liczba kategorii\n", "\n", "- Układ hipotez:\n", "\n", "$\\;\\;\\;\\;\\;H_0: p_i = p_i^0 \\textrm{ dla } i = 1, 2, ..., k$\n", "\n", "$\\;\\;\\;\\;\\;H_1: \\exists_i p_i \\neq p_i^0$\n", "\n", "- Statystyka testowa:\n", "\n", "$$\\chi^2=\\sum_{i=1}^{k}\\frac{(O_i-E_i)^2}{E_i}$$\n", "\n", "- Liczebności oczekiwane:\n", "\n", "$$E_i = p_i * N$$\n", "\n", "- Założenia:\n", "\n", "$\\;\\;\\;\\;\\;$niezależność obserwacji\n", "\n", "$\\;\\;\\;\\;\\;\\forall_i E_i > 5$\n", "\n", "- Stopnie swobody:\n", "\n", "$\\;\\;\\;\\;\\;df = k -1$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Zbiór krytyczny" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "image/svg+xml": [ "\n", "\n", "\n", "\n" ], "text/plain": [ "

" ] }, "metadata": { "needs_background": "light" }, "output_type": "display_data" } ], "source": [ "c_crit()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Test zgodności $\\chi^2$ - przykład" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Dane producenta:\n", "\n", "|Iron Man | Thor | Kapitan Ameryka|\n", "|---|---|---|\n", "|60\\% | 30\\% | 10\\% |\n", "\n", "Dane z próbki:\n", "\n", "|Iron Man | Thor | Kapitan Ameryka|\n", "|---|---|---|\n", "|45/100 | 50/100 | 5/100 |" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$\\alpha=0.05$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$H_0: p_1=0.6\\textrm{, }p_2=0.3\\textrm{, }p_3=0.1$\n", "\n", "$H_1: \\exists_i p_i \\neq p_i^0$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$E_1 = 100 * 0.6 = 60$\n", "\n", "$E_2 = 100 * 0.3 = 30$\n", "\n", "$E_3 = 100 * 0.1 = 10$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$\\chi^2=\\sum_{i=1}^{k}\\frac{(O_i-E_i)^2}{E_i} = \\frac{(45 - 60)^2}{60} + \\frac{(50 - 30)^2}{30} + \\frac{(5 - 10)^2}{10} = 19.58$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$P(\\chi^2>19.58) = 0.0001$\n", "\n", "Odrzucamy $H_0$ na rzecz $H_1$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Test niezależności $\\chi^2$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "- Układ hipotez:\n", "\n", "$\\;\\;\\;\\;\\;H_0: \\textrm{ zmienne są niezależne}$\n", " \n", "$\\;\\;\\;\\;\\;H_1: \\textrm{ zmienne są zależne}$\n", "\n", "- Statystyka testowa:\n", "\n", " $$\\chi^2=\\sum\\limits_{i=1}^{k}\\sum\\limits_{j=1}^{w}\\frac{(O_{ij}-E_{ij})^2}{E_{ij}}$$\n", " \n", "- Liczebności oczekiwane:\n", "\n", "$\\;\\;\\;\\;\\;E_{ij} = (W_i * K_j) / N$\n", "\n", "$\\;\\;\\;\\;\\;W_i$ - suma $i$-tego wiersza\n", "\n", "$\\;\\;\\;\\;\\;K_j$ - suma $j$-tej kolumny\n", "\n", "- Stopnie swobody:\n", "\n", "$\\;\\;\\;\\;\\;w$ - liczba wierszy \n", "\n", "$\\;\\;\\;\\;\\;k$ - liczba kolumn \n", "\n", "$\\;\\;\\;\\;\\;df = (w-1)*(k-1)$\n", "\n", "- Założenia:\n", "\n", "$\\;\\;\\;\\;\\;$niezależność obserwacji\n", "\n", "$\\;\\;\\;\\;\\;\\forall_{i,j} E_{ij} > 5$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Liczebności oczekiwane" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "| | k | !k | suma|\n", "|---| ----| --- |---|\n", "|p|30 | 0 | 30 |\n", "|!p|0 | 30 | 30 |\n", "|suma | 30 | 30 | 60 |" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "- niezależność zdarzeń: $P(A\\cap B)=P(A)P(B)$\n", "\n", "- $E_{ij} = N * p_{ij}$\n", "\n", "$$p_{i}=\\frac{W_i}{N}$$\n", "\n", "$$p_{j}=\\frac{K_j}{N}$$\n", "\n", "$$p_{ij}=\\frac{W_i}{N}\\frac{K_j}{N}$$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "| | k | !k | suma|\n", "|---| ----| --- |---|\n", "|p| 15 |15 | 30 |\n", "|!p| 15 | 15 | 30 |\n", "|suma | 30 | 30 | 60 |" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Test niezależności $\\chi^2$ - przykład" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "| | Sumo | Line Follower | Freestyle | Suma |\n", "|---|---|---|---|---|\n", "|Mężczyźni | 200 | 150 | 50 | 400 | \n", "|Kobiety | 250 | 300 | 50 | 600 | \n", "|Suma | 450 | 450 | 100 | 1000 | " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$\\alpha=0.05$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$H_0: \\textrm{ zmienne są niezależne}$\n", "\n", "$H_1: \\textrm{ zmienne są zależne}$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$df = (w - 1) * (k - 1) = (2 - 1) * (3 - 1) = 2$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$E_{ij} = (W_i * K_j) / N$\n", "\n", "$E_{11} = (400 * 450) / 1000 = 180000/1000 = 180$\n", "\n", "$E_{12} = (400 * 450) / 1000 = 180000/1000 = 180$\n", "\n", "$E_{13} = (400 * 100) / 1000 = 40000/1000 = 40$\n", "\n", "$E_{21} = (600 * 450) / 1000 = 270000/1000 = 270$\n", "\n", "$E_{22} = (600 * 450) / 1000 = 270000/1000 = 270$\n", "\n", "$E_{23} = (600 * 100) / 1000 = 60000/1000 = 60$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$\\chi^2=\\sum\\limits_{ij}^{}\\frac{(O_{ij}-E_{ij})^2}{E_{ij}}= \\frac{(200 - 180)^2}{180} + \\frac{(150 - 180)^2}{180} + \\frac{(50 - 40)^2}{40} + \\frac{(250 - 270)^2}{270} + \\frac{(300 - 270)^2}{270} + \\frac{(50 - 60)^2}{60} = 16.2$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$P(\\chi^2>16.2) = 0.0003$\n", "\n", "Odrzucamy $H_0$ na rzecz $H_1$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Poprawka Yatesa" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$$\\chi^2_{Yates}=\\sum_{i=1}^{k}\\frac{(|O_i-E_i|-0.5)^2}{E_i}$$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Miary siły zależności" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Współczynnik $\\phi$ Yule'a (dla 2x2)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ " $$\\phi=\\sqrt{\\frac{\\chi^2}{n}}$$\n", " \n", " Zakres: $[0,1]$\n", " \n", " $$\\phi=\\frac{n_{11}n_{00}-n_{10}n_{01}}{\\sqrt{(n_{11}+n_{10})(n_{11}+n_{01})(n_{00}+n_{10})(n_{00}+n_{01}})}$$\n", " \n", " Zakres: $[-1,1]$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Współczynnik kontyngencji C Pearsona" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$$C = \\sqrt{\\frac{\\chi^2}{\\chi^2+n}}$$\n", "\n", "Zakres: $\\left[0, \\sqrt{\\frac{\\textrm{min}(k-1, w-1)}{1+\\textrm{min}(k-1,w-1)}}\\right]$" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Współczynnik V Cramera" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "$$V = \\sqrt{\\frac{\\chi^2}{n\\cdot \\textrm{min}(k-1,w-1)}}$$\n", "\n", "Zakres: $[0, 1]$" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.9" }, "latex_envs": { "LaTeX_envs_menu_present": true, "autoclose": false, "autocomplete": true, "bibliofile": "biblio.bib", "cite_by": "apalike", "current_citInitial": 1, "eqLabelWithNumbers": true, "eqNumInitial": 1, "hotkeys": { "equation": "Ctrl-E", "itemize": "Ctrl-I" }, "labels_anchors": false, "latex_user_defs": false, "report_style_numbering": false, "user_envs_cfg": false } }, "nbformat": 4, "nbformat_minor": 4 }