Saturday, November 18, 2017

Install and configure jupter for pyspark

[root@cdh-vm ~]# pip install jupyter


[donghua@cdh-vm ~]$ grep PYSPARK  ~/.bash_profile
export PYSPARK_DRIVER_PYTHON=ipython
export PYSPARK_DRIVER_PYTHON_OPTS="notebook --ip 192.168.56.10 --port 3333 --no-mathjax"

[donghua@cdh-vm ~]$ pyspark
[TerminalIPythonApp] WARNING | Subcommand `ipython notebook` is deprecated and will be removed in future versions.
[TerminalIPythonApp] WARNING | You likely want to use `jupyter notebook` in the future
[I 13:34:34.137 NotebookApp] Serving notebooks from local directory: /home/donghua
[I 13:34:34.137 NotebookApp] 0 active kernels
[I 13:34:34.137 NotebookApp] The Jupyter Notebook is running at:
[I 13:34:34.137 NotebookApp] http://192.168.56.10:3333/?token=3a127150007c4f0816871644a97feb3b2c1ad721411e9576
[I 13:34:34.137 NotebookApp] Use Control-C to stop this server and shut down all kernels (twice to skip confirmation).
[W 13:34:34.138 NotebookApp] No web browser found: could not locate runnable browser.
[C 13:34:34.138 NotebookApp]

    Copy/paste this URL into your browser when you connect for the first time,
    to login with a token:
        http://192.168.56.10:3333/?token=3a127150007c4f0816871644a97feb3b2c1ad721411e9576

image

================================================

[root@cdh-vm ~]# pip install jupyter
Collecting jupyter
  Downloading jupyter-1.0.0-py2.py3-none-any.whl
Collecting nbconvert (from jupyter)
  Downloading nbconvert-5.3.1-py2.py3-none-any.whl (387kB)
    100% |████████████████████████████████| 389kB 723kB/s
Collecting ipywidgets (from jupyter)
  Downloading ipywidgets-7.0.5-py2.py3-none-any.whl (68kB)
    100% |████████████████████████████████| 71kB 3.2MB/s
Collecting notebook (from jupyter)
  Downloading notebook-5.2.1-py2.py3-none-any.whl (8.0MB)
    100% |████████████████████████████████| 8.0MB 126kB/s
Collecting qtconsole (from jupyter)
  Downloading qtconsole-4.3.1-py2.py3-none-any.whl (108kB)
    100% |████████████████████████████████| 112kB 4.3MB/s
Collecting jupyter-console (from jupyter)
  Downloading jupyter_console-5.2.0-py2.py3-none-any.whl
Collecting ipykernel (from jupyter)
  Downloading ipykernel-4.6.1-py2-none-any.whl (104kB)
    100% |████████████████████████████████| 112kB 5.6MB/s
Collecting pandocfilters>=1.4.1 (from nbconvert->jupyter)
  Downloading pandocfilters-1.4.2.tar.gz
Collecting entrypoints>=0.2.2 (from nbconvert->jupyter)
  Downloading entrypoints-0.2.3-py2.py3-none-any.whl
Collecting jinja2 (from nbconvert->jupyter)
  Downloading Jinja2-2.10-py2.py3-none-any.whl (126kB)
    100% |████████████████████████████████| 133kB 3.6MB/s
Collecting testpath (from nbconvert->jupyter)
  Downloading testpath-0.3.1-py2.py3-none-any.whl (161kB)
    100% |████████████████████████████████| 163kB 2.6MB/s
Collecting mistune>=0.7.4 (from nbconvert->jupyter)
  Downloading mistune-0.8.1-py2.py3-none-any.whl
Collecting nbformat>=4.4 (from nbconvert->jupyter)
  Downloading nbformat-4.4.0-py2.py3-none-any.whl (155kB)
    100% |████████████████████████████████| 163kB 2.9MB/s
Requirement already satisfied: pygments in /usr/lib64/python2.7/site-packages (from nbconvert->jupyter)
Collecting bleach (from nbconvert->jupyter)
  Downloading bleach-2.1.1-py2.py3-none-any.whl
Requirement already satisfied: traitlets>=4.2 in /usr/lib/python2.7/site-packages (from nbconvert->jupyter)
Collecting jupyter-core (from nbconvert->jupyter)
  Downloading jupyter_core-4.4.0-py2.py3-none-any.whl (126kB)
    100% |████████████████████████████████| 133kB 3.5MB/s
Requirement already satisfied: ipython<6.0.0,>=4.0.0; python_version < "3.3" in /usr/lib/python2.7/site-packages (from ipywidgets->jupyter)
Collecting widgetsnbextension~=3.0.0 (from ipywidgets->jupyter)
  Downloading widgetsnbextension-3.0.8-py2.py3-none-any.whl (2.2MB)
    100% |████████████████████████████████| 2.2MB 347kB/s
Requirement already satisfied: ipython-genutils in /usr/lib/python2.7/site-packages (from notebook->jupyter)
Collecting jupyter-client (from notebook->jupyter)
  Downloading jupyter_client-5.1.0-py2.py3-none-any.whl (84kB)
    100% |████████████████████████████████| 92kB 2.8MB/s
Collecting tornado>=4 (from notebook->jupyter)
  Downloading tornado-4.5.2.tar.gz (483kB)
    100% |████████████████████████████████| 491kB 1.8MB/s
Collecting terminado>=0.3.3; sys_platform != "win32" (from notebook->jupyter)
  Downloading terminado-0.7-py2.py3-none-any.whl
Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.0 in /usr/lib/python2.7/site-packages (from jupyter-console->jupyter)
Collecting configparser>=3.5; python_version == "2.7" (from entrypoints>=0.2.2->nbconvert->jupyter)
  Downloading configparser-3.5.0.tar.gz
Collecting MarkupSafe>=0.23 (from jinja2->nbconvert->jupyter)
  Downloading MarkupSafe-1.0.tar.gz
Collecting jsonschema!=2.5.0,>=2.4 (from nbformat>=4.4->nbconvert->jupyter)
  Downloading jsonschema-2.6.0-py2.py3-none-any.whl
Collecting html5lib!=1.0b1,!=1.0b2,!=1.0b3,!=1.0b4,!=1.0b5,!=1.0b6,!=1.0b7,!=1.0b8,>=0.99999999pre (from bleach->nbconvert->jupyter)
  Downloading html5lib-1.0b10-py2.py3-none-any.whl (112kB)
    100% |████████████████████████████████| 112kB 3.7MB/s
Requirement already satisfied: six in /usr/lib/python2.7/site-packages (from bleach->nbconvert->jupyter)
Requirement already satisfied: decorator in /usr/lib/python2.7/site-packages (from traitlets>=4.2->nbconvert->jupyter)
Requirement already satisfied: enum34; python_version == "2.7" in /usr/lib/python2.7/site-packages (from traitlets>=4.2->nbconvert->jupyter)
Requirement already satisfied: pexpect; sys_platform != "win32" in /usr/lib/python2.7/site-packages (from ipython<6.0.0,>=4.0.0; python_version < "3.3"->ipywidgets->jupyter)
Requirement already satisfied: backports.shutil-get-terminal-size; python_version == "2.7" in /usr/lib/python2.7/site-packages (from ipython<6.0.0,>=4.0.0; python_version < "3.3"->ipywidgets->jupyter)
Requirement already satisfied: setuptools>=18.5 in /usr/lib/python2.7/site-packages (from ipython<6.0.0,>=4.0.0; python_version < "3.3"->ipywidgets->jupyter)
Requirement already satisfied: pathlib2; python_version == "2.7" or python_version == "3.3" in /usr/lib/python2.7/site-packages (from ipython<6.0.0,>=4.0.0; python_version < "3.3"->ipywidgets->jupyter)
Requirement already satisfied: simplegeneric>0.8 in /usr/lib/python2.7/site-packages (from ipython<6.0.0,>=4.0.0; python_version < "3.3"->ipywidgets->jupyter)
Requirement already satisfied: pickleshare in /usr/lib/python2.7/site-packages (from ipython<6.0.0,>=4.0.0; python_version < "3.3"->ipywidgets->jupyter)
Collecting pyzmq>=13 (from jupyter-client->notebook->jupyter)
  Downloading pyzmq-16.0.3-cp27-cp27mu-manylinux1_x86_64.whl (3.0MB)
    100% |████████████████████████████████| 3.0MB 299kB/s
Collecting python-dateutil>=2.1 (from jupyter-client->notebook->jupyter)
  Downloading python_dateutil-2.6.1-py2.py3-none-any.whl (194kB)
    100% |████████████████████████████████| 194kB 2.6MB/s
Requirement already satisfied: backports.ssl_match_hostname in /usr/lib/python2.7/site-packages (from tornado>=4->notebook->jupyter)
Collecting singledispatch (from tornado>=4->notebook->jupyter)
  Downloading singledispatch-3.4.0.3-py2.py3-none-any.whl
Collecting certifi (from tornado>=4->notebook->jupyter)
  Downloading certifi-2017.11.5-py2.py3-none-any.whl (330kB)
    100% |████████████████████████████████| 337kB 596kB/s
Collecting backports_abc>=0.4 (from tornado>=4->notebook->jupyter)
  Downloading backports_abc-0.5-py2.py3-none-any.whl
Requirement already satisfied: ptyprocess in /usr/lib/python2.7/site-packages (from terminado>=0.3.3; sys_platform != "win32"->notebook->jupyter)
Requirement already satisfied: wcwidth in /usr/lib/python2.7/site-packages (from prompt-toolkit<2.0.0,>=1.0.0->jupyter-console->jupyter)
Collecting functools32; python_version == "2.7" (from jsonschema!=2.5.0,>=2.4->nbformat>=4.4->nbconvert->jupyter)
  Downloading functools32-3.2.3-2.zip
Collecting webencodings (from html5lib!=1.0b1,!=1.0b2,!=1.0b3,!=1.0b4,!=1.0b5,!=1.0b6,!=1.0b7,!=1.0b8,>=0.99999999pre->bleach->nbconvert->jupyter)
  Downloading webencodings-0.5.1-py2.py3-none-any.whl
Requirement already satisfied: scandir; python_version < "3.5" in /usr/lib64/python2.7/site-packages (from pathlib2; python_version == "2.7" or python_version == "3.3"->ipython<6.0.0,>=4.0.0; python_version < "3.3"->ipywidgets->jupyter)
Installing collected packages: pandocfilters, configparser, entrypoints, MarkupSafe, jinja2, testpath, mistune, functools32, jsonschema, jupyter-core, nbformat, webencodings, html5lib, bleach, nbconvert, pyzmq, python-dateutil, jupyter-client, singledispatch, certifi, backports-abc, tornado, terminado, ipykernel, notebook, widgetsnbextension, ipywidgets, qtconsole, jupyter-console, jupyter
  Running setup.py install for pandocfilters ... done
  Running setup.py install for configparser ... done
  Running setup.py install for MarkupSafe ... done
  Running setup.py install for functools32 ... done
  Running setup.py install for tornado ... done
Successfully installed MarkupSafe-1.0 backports-abc-0.5 bleach-2.1.1 certifi-2017.11.5 configparser-3.5.0 entrypoints-0.2.3 functools32-3.2.3.post2 html5lib-1.0b10 ipykernel-4.6.1 ipywidgets-7.0.5 jinja2-2.10 jsonschema-2.6.0 jupyter-1.0.0 jupyter-client-5.1.0 jupyter-console-5.2.0 jupyter-core-4.4.0 mistune-0.8.1 nbconvert-5.3.1 nbformat-4.4.0 notebook-5.2.1 pandocfilters-1.4.2 python-dateutil-2.6.1 pyzmq-16.0.3 qtconsole-4.3.1 singledispatch-3.4.0.3 terminado-0.7 testpath-0.3.1 tornado-4.5.2 webencodings-0.5.1 widgetsnbextension-3.0.8
[root@cdh-vm ~]#