From 359bbe42c3dd1d6238aca1776c145feb1a19158a Mon Sep 17 00:00:00 2001 From: Greg Zynda Date: Fri, 26 Jan 2018 13:38:31 -0500 Subject: [PATCH 1/7] Renamed sd2e to singularity --- images/base/Dockerfile | 3 ++- images/{sd2e => singularity}/Dockerfile | 0 images/{sd2e => singularity}/dotjupyter/custom.css | 0 3 files changed, 2 insertions(+), 1 deletion(-) rename images/{sd2e => singularity}/Dockerfile (100%) rename images/{sd2e => singularity}/dotjupyter/custom.css (100%) diff --git a/images/base/Dockerfile b/images/base/Dockerfile index c49f644..bf1f3cb 100644 --- a/images/base/Dockerfile +++ b/images/base/Dockerfile @@ -51,7 +51,8 @@ cd /root && \ rm -rf sd2e-jupyter-ascending && \ # Ubuntu Packages apt-get update && \ - apt-get install -y tree less && \ + # jq is required for abaco CLI + apt-get install -y tree less jq && \ #apt-get install -y tree less openjdk-8-jre ca-certificates-java && \ apt-get clean && \ # Replace stock Agave CLI with SD2E-customized tooling diff --git a/images/sd2e/Dockerfile b/images/singularity/Dockerfile similarity index 100% rename from images/sd2e/Dockerfile rename to images/singularity/Dockerfile diff --git a/images/sd2e/dotjupyter/custom.css b/images/singularity/dotjupyter/custom.css similarity index 100% rename from images/sd2e/dotjupyter/custom.css rename to images/singularity/dotjupyter/custom.css From 1d39563dd4d18b714fdff334b8cad3293f9b42e2 Mon Sep 17 00:00:00 2001 From: Greg Zynda Date: Sat, 27 Jan 2018 18:42:56 -0500 Subject: [PATCH 2/7] Testing singularity --- Makefile | 12 ++++ build/build_jupyteruser.sh | 10 +-- images/base/Dockerfile | 3 +- images/custom/Dockerfile | 4 +- images/singularity/Dockerfile | 80 +++++++++++------------- images/singularity/dotjupyter/custom.css | 0 6 files changed, 56 insertions(+), 53 deletions(-) delete mode 100644 images/singularity/dotjupyter/custom.css diff --git a/Makefile b/Makefile index 23f6698..fc0484a 100644 --- a/Makefile +++ b/Makefile @@ -53,12 +53,16 @@ test: docker case $$TARGET in base) build/build_jupyteruser.sh test images/base + ;; sd2e) build/build_jupyteruser.sh test images/custom + ;; singularity) build/build_jupyteruser.sh test images/singularity + ;; *) $(MAKE) help + ;; esac stage: docker @@ -66,12 +70,16 @@ stage: docker case $$TARGET in base) build/build_jupyteruser.sh stage images/base + ;; sd2e) build/build_jupyteruser.sh stage images/custom + ;; singularity) build/build_jupyteruser.sh stage images/singularity + ;; *) $(MAKE) help + ;; esac release: docker @@ -79,12 +87,16 @@ release: docker case $$TARGET in base) build/build_jupyteruser.sh release images/base + ;; sd2e) build/build_jupyteruser.sh release images/custom + ;; singularity) build/build_jupyteruser.sh release images/singularity + ;; *) $(MAKE) help + ;; esac clean: docker diff --git a/build/build_jupyteruser.sh b/build/build_jupyteruser.sh index bf30f43..e73d280 100755 --- a/build/build_jupyteruser.sh +++ b/build/build_jupyteruser.sh @@ -145,11 +145,11 @@ function pushImage { fi fi fi - if askTrue "Do you want to tag ${IMG}:${VERSION} as ${3}?"; then - ed "docker tag ${IMG}:${VERSION} ${IMG}:${3}" - docker tag ${IMG}:${VERSION} ${IMG}:${3} - ed "docker push ${IMG}:${3}" - docker push ${IMG}:${3} + if askTrue "Do you want to tag ${IMG}:${VERSION} as ${2}?"; then + ed "docker tag ${IMG}:${VERSION} ${IMG}:${2}" + docker tag ${IMG}:${VERSION} ${IMG}:${2} + ed "docker push ${IMG}:${2}" + docker push ${IMG}:${2} fi } diff --git a/images/base/Dockerfile b/images/base/Dockerfile index bf1f3cb..5bb3a3d 100644 --- a/images/base/Dockerfile +++ b/images/base/Dockerfile @@ -1,5 +1,5 @@ #Image: sd2e/jupyteruser-base -#Version: 0.1.0 +#Version: 0.1.1 FROM taccsciapps/jupyteruser-base:latest LABEL origin="https://bitbucket.org/gzynda/jupyter/src/40adbad7b10b34de715d5a86aec3fcf65e724b70/notebooks/base/jupyter-notebook/Dockerfile?at=CIC-425-image-redesign&fileviewer=file-view-default" @@ -59,6 +59,7 @@ apt-get update && \ curl -L https://raw.githubusercontent.com/sd2e/sd2e-cli/master/install/install.sh | sh && \ rm -rf /opt/cli && \ mv /home/jupyter/sd2e-cloud-cli /opt/cli && \ + chmod -R a+rX /opt/cli && \ # Spark 2.2.1 and pyspark #cd /usr/local && \ # curl http://apache.claz.org/spark/spark-2.2.1/spark-2.2.1-bin-hadoop2.7.tgz | tar -xzf - && \ diff --git a/images/custom/Dockerfile b/images/custom/Dockerfile index 9e31b4e..caf599b 100644 --- a/images/custom/Dockerfile +++ b/images/custom/Dockerfile @@ -1,7 +1,7 @@ #Image: sd2e/jupyteruser-sd2e -#Version: 0.3.0 +#Version: 0.3.1 -FROM sd2e/jupyteruser-base:0.1.0 +FROM sd2e/jupyteruser-base:0.1.1 # Image release USER root diff --git a/images/singularity/Dockerfile b/images/singularity/Dockerfile index 45539c1..85e49bb 100644 --- a/images/singularity/Dockerfile +++ b/images/singularity/Dockerfile @@ -1,50 +1,40 @@ -FROM sd2e/jupyteruser-custom:devel +#Image: sd2e/jupyteruser-singularity +#Version: 0.1.0 -# Convention envs to help -# people build portable scripts -ENV WORK /home/jupyter/tacc-work -ENV SD2_DATA /home/jupyter/sd2e-community +FROM sd2e/jupyteruser-sd2e:0.3.1 -# Force ownership and permissions on /home/jupyter +# Image release USER root -RUN echo 'PS1="$JPY_USER@\h:\w\$ "' >> /home/jupyter/.bashrc -ADD ./dotjupyter/custom.css /home/jupyter/.jupyter/custom/custom.css -ADD docs/SD2E_README.md /home/jupyter/SD2E_README.md -RUN chmod -R a+rw /home -RUN chown -R jupyter /home/jupyter/examples - -# Bioconductor -RUN conda install -c bioconda bioconductor-biocinstaller -# BioPython -RUN conda install -c anaconda biopython - -# Install latest AgavePy -WORKDIR /home/jupyter/ -RUN /opt/conda/envs/python2/bin/pip install -e git+https://github.com/TACC/agavepy.git#egg=agavepy -RUN pip install -e git+https://github.com/TACC/agavepy.git#egg=agavepy - -# Enable sd2nb sharing service -RUN cd /root && \ - git clone https://github.com/SD2E/sd2nb-app.git && \ - cd sd2nb-app && \ - make && make install - -# Enable sd2e-jupyter -RUN cd /root && \ - git clone https://github.com/SD2E/sd2e-jupyter-ascending.git && \ - cd sd2e-jupyter-ascending && \ - make && make install - -# Tree -RUN apt-get update && \ - apt-get install -y tree && \ - apt-get clean - -# Make sure Jupyter group == SD2E-Community == 819382 -RUN groupadd --gid 819382 G-819382 -RUN usermod -g G-819382 jupyter -a -G G-818301 -RUN find /home/jupyter -group G-819382 -exec chown jupyter:G-819382 {} \; - -ADD jupyteruser-sd2e_release /etc/jupyteruser-sd2e_release +ARG image_version=development +RUN echo "${image_version}" > /etc/sd2e_jupyteruser-sd2e-release && \ + chmod a+r /etc/sd2e_jupyteruser-sd2e-release + +# Make sd2e dir +ARG SD2E_DIR=/opt/sd2e +ARG SD2E_HOME=/opt/sd2e/jupyter +RUN [ ! -e $SD2E_DIR ] && mkdir -p $SD2E_HOME && \ +########################################## +# Archive HOME +########################################## +cp -r examples SD2E_README.md $SD2E_HOME/ && \ +#cp -r .cache $SD2E_HOME/ && \ +tar -czf $SD2E_DIR/home_archive.tar.gz -C $SD2E_DIR jupyter && \ +chmod 644 $SD2E_DIR/home_archive.tar.gz && \ +rm -rf $SD2E_HOME +########################################## +# Modify default config +########################################## +RUN echo -e "\n\n# Set a password if PASSWORD is set\n\ +if 'PASSWORD' in os.environ:\n\ +\tfrom IPython.lib import passwd\n\ +\tc.NotebookApp.password = passwd(os.environ['PASSWORD'])\n\ +\tdel os.environ['PASSWORD']\n\n\ +c.NotebookApp.iopub_data_rate_limit=10000000" >> /etc/jupyter/jupyter_notebook_config.py +########################################## +# Update CMD +########################################## +COPY enter_singularity.sh /bin/enter_singularity.sh +RUN chmod 755 /bin/enter_singularity.sh +CMD ["enter_singularity.sh"] USER jupyter diff --git a/images/singularity/dotjupyter/custom.css b/images/singularity/dotjupyter/custom.css deleted file mode 100644 index e69de29..0000000 From 3d036346ca1b907be018af57a6d7898f72879a61 Mon Sep 17 00:00:00 2001 From: Greg Zynda Date: Thu, 1 Feb 2018 09:49:29 -0500 Subject: [PATCH 3/7] Added PYTHONUSERBASE for persistant python packages, fixed git, and resolved issues 8 and 9 --- images/base/Dockerfile | 39 +++++++++-------- images/base/src/configuration.rc | 56 +++++++++++++++++++++++++ images/custom/Dockerfile | 28 ++++++++++--- images/singularity/Dockerfile | 21 +++++++--- images/singularity/enter_singularity.sh | 31 ++++++++++++++ 5 files changed, 145 insertions(+), 30 deletions(-) create mode 100644 images/base/src/configuration.rc create mode 100644 images/singularity/enter_singularity.sh diff --git a/images/base/Dockerfile b/images/base/Dockerfile index 5bb3a3d..2874eac 100644 --- a/images/base/Dockerfile +++ b/images/base/Dockerfile @@ -1,7 +1,7 @@ #Image: sd2e/jupyteruser-base #Version: 0.1.1 -FROM taccsciapps/jupyteruser-base:latest +FROM taccsciapps/jupyteruser-base:0.1.2 LABEL origin="https://bitbucket.org/gzynda/jupyter/src/40adbad7b10b34de715d5a86aec3fcf65e724b70/notebooks/base/jupyter-notebook/Dockerfile?at=CIC-425-image-redesign&fileviewer=file-view-default" ############################################### @@ -11,20 +11,21 @@ USER root # Image release ARG image_version=development +ENV NB_GID=G-819382 RUN echo "${image_version}" > /etc/sd2e_jupyteruser-base-release && \ chmod a+r /etc/sd2e_jupyteruser-base-release && \ # Make sure Jupyter group == SD2E-Community == 819382 -groupadd --gid 819382 G-819382 && \ -usermod -a -G G-819382 jupyter && \ -usermod -g G-819382 jupyter && \ +groupadd --gid 819382 $NB_GID && \ +usermod -a -G $NB_GID jupyter && \ +usermod -g $NB_GID jupyter && \ # Jupyterhub correctly sets JUPYTERHUB_USER echo 'PS1="$JUPYTERHUB_USER@\h:\w\$ "' >> /home/jupyter/.bashrc && \ # allow users to add kernels -chmod 777 /opt/conda/share/jupyter/kernels - +chmod 777 /opt/conda/share/jupyter/kernels && \ # Environment variables -ENV WORK /home/jupyter/tacc-work -ENV SD2_DATA /home/jupyter/sd2e-community +echo "WORK=/home/jupyter/tacc-work\n\ +SD2_DATA=/home/jupyter/sd2e-community\n\ +PYTHONUSERBASE=/home/jupyter/tacc-work/jupyter_packages\n" >> /home/jupyter/.bashrc # Add readme ADD docs/SD2E_README.md /home/jupyter/SD2E_README.md @@ -52,27 +53,25 @@ cd /root && \ # Ubuntu Packages apt-get update && \ # jq is required for abaco CLI - apt-get install -y tree less jq && \ + apt-get install -y tree less jq rsync column && \ #apt-get install -y tree less openjdk-8-jre ca-certificates-java && \ - apt-get clean && \ + docker-clean # Replace stock Agave CLI with SD2E-customized tooling -curl -L https://raw.githubusercontent.com/sd2e/sd2e-cli/master/install/install.sh | sh && \ - rm -rf /opt/cli && \ +ADD src/configuration.rc configuration.rc +RUN git clone https://github.com/SD2E/sd2e-cli.git && \ + mv configuration.rc sd2e-cli/ && cd sd2e-cli && \ + git submodule update --init --recursive && make && \ + make install && rm -rf /opt/cli && rm -rf $HOME/sd2e-cli && \ mv /home/jupyter/sd2e-cloud-cli /opt/cli && \ - chmod -R a+rX /opt/cli && \ + chmod -R a+rX /opt/cli && chmod -R a+x /opt/cli/bin && \ # Spark 2.2.1 and pyspark #cd /usr/local && \ # curl http://apache.claz.org/spark/spark-2.2.1/spark-2.2.1-bin-hadoop2.7.tgz | tar -xzf - && \ # mv spark-2.2.1-bin-hadoop2.7 spark && \ # conda install --yes pyspark && \ # conda install -n python2 --yes pyspark && \ -# Clean conda cache -conda clean -tipsy && \ -# Clean temporary files -apt-get clean && conda clean -tipsy && \ -bash -c "for dir in /tmp/* /var/tmp/* /home/jupyter/.cache/pip /root/.cache/pip; do [ -e $dir ] && rm -rf $dir; done" - -ENV SPARK_HOME=/usr/local/spark +docker-clean +#ENV SPARK_HOME=/usr/local/spark ############################################### # Permissions diff --git a/images/base/src/configuration.rc b/images/base/src/configuration.rc new file mode 100644 index 0000000..7363fa7 --- /dev/null +++ b/images/base/src/configuration.rc @@ -0,0 +1,56 @@ +# SD2E TACC.Cloud variables +# +# Human-readable name of your TACC Cloud API tenant + +# Source this before running any make commands + +# Human-readable name for your project +export TENANT_NAME="DARPA SD2E" + +# Tenant shortname (see https://tacc.cloud/tenants for options) +export TENANT_KEY=sd2e + +# Namespace for custom executable assets +export TENANT_NS=sd2e + +# TACC Community project (for access to default resources) +export TENANT_PROJECT=SD2E-Community + +# Docker +# Dockerfile for building the CLI image +export TENANT_DOCKERFILE=Dockerfile +# DockerHub virtual organization +export TENANT_DOCKER_ORG=sd2e +# Docker image tag +export TENANT_DOCKER_TAG=cloud-cli +# Docker image version +# This comes from the VERSION file + +# Destination repository +# Repository and branch for the tenant-specific project +export TENANT_CLI_REPO=https://github.com/SD2E/sd2e-cli +export TENANT_CLI_BRANCH=master + +# Optional: Set a remote repo to populate the docs/ directory +export CLI_DOCS= +export CLI_DOCS_BRANCH= +export CLI_DOCS_DEST=docs + +# Optional: Set a remote repo to populate the extras directory +export CLI_EXTRAS_REPO= +export CLI_EXTRAS_BRANCH= +export CLI_EXTRAS_DEST=extras + +# Dont' change below this line unless you know what you're doing! + +# Submodules +export CLI_BASE_REPO=https://bitbucket.org/tacc-cic/cli +export CLI_BASE_BRANCH=master +export CLI_BASE_DEST=tacc-cli-base + +export ABACO_BASE_REPO=https://github.com/TACC-Reactors/abaco-cli.git +export ABACO_BASE_BRANCH=master +export ABACO_BASE_DEST=abaco-cli + +# Make support +export MAKE_OBJ=sd2e-cloud-cli diff --git a/images/custom/Dockerfile b/images/custom/Dockerfile index caf599b..a8667dd 100644 --- a/images/custom/Dockerfile +++ b/images/custom/Dockerfile @@ -16,7 +16,7 @@ USER root LABEL "comment"="Support for plotly igraph and networkx" "issue"="SD2E/jupyteruser-sd2e/issues/3" RUN conda install --quiet --yes python-igraph plotly networkx && \ conda install --quiet --yes -n python2 python-igraph plotly networkx && \ - conda clean -tipsy + docker-clean ########################################## # Probcomp stack @@ -43,8 +43,7 @@ bash -c "source activate python2 && pip2 install git+https://github.com/probcomp pip2 install git+https://github.com/probcomp/bayeslite.git@v0.3.2 && \ pip2 install git+https://github.com/probcomp/iventure.git@v0.2.2" && \ # Clean cache -apt-get clean && conda clean -tipsy && \ -bash -c "for dir in /tmp/* /var/tmp/* /home/jupyter/.cache/pip /root/.cache/pip; do [ -e $dir ] && rm -rf $dir; done" +docker-clean RUN mkdir -p /home/jupyter/examples/probcomp && \ cd /home/jupyter/examples/probcomp && \ @@ -89,12 +88,31 @@ COPY common-lisp/sbclrc /home/jupyter/.sbclrc # Move the kernel json file to the correct location RUN mkdir /opt/conda/share/jupyter/kernels/lisp && \ - mv /home/jupyter/.ipython/kernels/lisp/kernel.json /opt/conda/share/jupyter/kernels/lisp && \ + mv $HOME/.ipython/kernels/lisp/kernel.json /opt/conda/share/jupyter/kernels/lisp && \ chmod -R a+rwX /opt/conda/share/jupyter/kernels/lisp && \ - rm -rf /home/jupyter/.ipython + rm -rf ${HOME}/.ipython && docker-clean COPY examples/common-lisp /home/jupyter/examples/common-lisp +########################################## +# pyemd support +########################################## +USER root +LABEL "comment"="support for pyemd" "issue"="SD2E/jupyteruser-sd2e/issues/8" +RUN conda install --quiet --yes pyemd && \ + conda install --quiet --yes -n python2 pyemd && \ + docker-clean + +########################################## +# Support DSGRN code +########################################## +USER root +LABEL "comment"="Support DSGRN code" "issue"="SD2E/jupyteruser-sd2e/issues/9" +RUN apt-get install -y cmake graphviz && \ + conda install --quiet --yes python-graphviz && \ + conda install --quiet --yes -n python2 python-graphviz && \ + docker-clean + ############################################### # Permissions ############################################### diff --git a/images/singularity/Dockerfile b/images/singularity/Dockerfile index 85e49bb..0aaa260 100644 --- a/images/singularity/Dockerfile +++ b/images/singularity/Dockerfile @@ -6,11 +6,11 @@ FROM sd2e/jupyteruser-sd2e:0.3.1 # Image release USER root ARG image_version=development -RUN echo "${image_version}" > /etc/sd2e_jupyteruser-sd2e-release && \ - chmod a+r /etc/sd2e_jupyteruser-sd2e-release +RUN echo "${image_version}" > /etc/sd2e_jupyteruser-singularity-release && \ + chmod a+r /etc/sd2e_jupyteruser-singularity-release # Make sd2e dir -ARG SD2E_DIR=/opt/sd2e +ENV SD2E_DIR=/opt/sd2e ARG SD2E_HOME=/opt/sd2e/jupyter RUN [ ! -e $SD2E_DIR ] && mkdir -p $SD2E_HOME && \ ########################################## @@ -24,7 +24,7 @@ rm -rf $SD2E_HOME ########################################## # Modify default config ########################################## -RUN echo -e "\n\n# Set a password if PASSWORD is set\n\ +RUN echo "\n\n# Set a password if PASSWORD is set\n\ if 'PASSWORD' in os.environ:\n\ \tfrom IPython.lib import passwd\n\ \tc.NotebookApp.password = passwd(os.environ['PASSWORD'])\n\ @@ -35,6 +35,17 @@ c.NotebookApp.iopub_data_rate_limit=10000000" >> /etc/jupyter/jupyter_notebook_c ########################################## COPY enter_singularity.sh /bin/enter_singularity.sh RUN chmod 755 /bin/enter_singularity.sh -CMD ["enter_singularity.sh"] +#ENTRYPOINT ["/bin/bash"] +#CMD ["/bin/enter_singularity.sh"] +########################################## +# Clean up ENV +########################################## +ENV NB_USER= +ENV NB_UID= +ENV NB_GID= +#ENV HOME= +#ENV USER= +#ENV XDG_CACHE_HOME= +#ENV WORK= USER jupyter diff --git a/images/singularity/enter_singularity.sh b/images/singularity/enter_singularity.sh new file mode 100644 index 0000000..162e199 --- /dev/null +++ b/images/singularity/enter_singularity.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Check to see if work defined +if [ -n "$STOCKYARD" ]; then + # Work is defined + if [ -e $STOCKYARD/jupyter ]; then + echo "Example notebooks already detected. To refresh them, delete\n \$STOCKYARD/jupyter" + else + echo "Unpacking example notebooks" + tar -xzf $SD2E_DIR/home_archive.tar.gz -C $STOCKYARD + fi +else + echo "No \$STOCKYARD detected - Not unpacking example notebooks" +fi + +# Unpack jupyter config +#[ ! -e $HOME/.jupyter ] && tar -xzf /usr/share/sd2e/dotjupyter.tar.gz -C $HOME + +# Change to jupyter directory if it exists +if [ -n "$STOCKYARD" ]; then + echo "Changing to SD2E notebook directory" + cd $STOCKYARD/jupyter +fi + +# Delete any legacy configs +if [ -e $HOME/.jupyter ]; then + rm -rf $HOME/.jupyter +fi + +# Start the notebook +start-notebook.sh From 00a3a0e56543991dcec8f78024200e7a067bd820 Mon Sep 17 00:00:00 2001 From: Greg Zynda Date: Thu, 1 Feb 2018 15:25:46 -0500 Subject: [PATCH 4/7] Added jpy_user for sd2nb --- CHANGELOG.md | 10 +++++ CONTRIBUTING.md | 92 ++++++++++++++++++++++++++++++++-------- README.md | 16 +++++-- images/base/Dockerfile | 11 ++--- images/custom/Dockerfile | 3 +- 5 files changed, 103 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e882ff4..43a10bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ All notable changes to this project are to be documented in this file. +## Version 0.3.1 + + - Added `PYTHONUSERBASE` variable, which points at `tacc-work/jupyter_packages`, and allows for persistent local pip installs + - Added pyemd + - Added graphviz, python-graphviz, cmake to support DSGRN + - FIXED git funtionality by setting + - `GIT_COMMITTER\_NAME=$JUPYTERHUB\_USER` + - `GIT_COMMITTER\_EMAIL=${JUPYTERHUB\_USER}@sd2e.org` + - REMOVED the hardcoding of `MPLBACKEND` to `Agg` + ## Version 0.3.0 - Refactored images to start from taccsciapps/jupyteruser-base diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f81c9b1..51bb981 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,40 +1,96 @@ # Contribute to jupyteruser-sd2e -Have an idea for what our base Jupyter Notebooks environment needs in it? Here's how to make it happen! +If you find that the SD2E jupyter environment lacks some + + - tools + - libraries + - packages + - examples + +that the community would benefit from, here is a guide on getting it incorporated. ## Propose work -* Raise an issue: - * Describing a proposed change/update/improvement/fix (we provide a nice template). - * If you have one (even just an idea) provide an implementation plan - * Include thoughts about how to test the new feature, as well as how it will be communicated to end users +* Raise an issue \([link](https://github.com/SD2E/jupyteruser-sd2e/issues)\): + * Describing a proposed change/update/improvement/fix (we provide a nice template). + * List and link the software \(required\) + * boost >= 1.58 - (http://www.boost.org/users/download/) + * python boost >= 0.1 + * List any known dependencies + * C++ + * If you have one (even just an idea) provide an implementation plan + * `apt-get install foo` + * `pip install foo` + * Let us know how we can verify the software installation + * Sample commands + * Example notebooks ## Contribute work -* To contribute, fork this repo and pull it down locally -* Checkout a new branch (myFeature) off `development` named after the Github issue documenting the bug you're fixing or feature you're adding +While not necessary, you can also locally develop and test your own feature for integration. When you are satisfied, you can then submit a pull request for integration into the main SD2E repository. + +### Fork the repository + +While the project is open, we restrict the ability to commit code. Begin by forking + +![fork](https://upload.wikimedia.org/wikipedia/commons/3/38/GitHub_Fork_Button.png) + +our repository to your own user account on github. This is necessary so we can see the remote repository to pull your changes from. Github cannot pull from your laptop. + +### Clone YOUR repository + +Clone a local copy of **your** repository + +``` +git clone https://github.com/[your_name_hear]/jupyteruser-sd2e.git +``` + +and then make a new branch (`myFeature`) for your feature based on the `development` branch. + ``` git checkout -b myFeature development ``` -* In `images/custom`: - * Edit the `Dockerfile` to add your software - * Add example files and notebooks -* Clear local image cache + +### Incorporate your changes + +Please add any external example or configuration files in a descriptive folder in + +``` +images/custom/ ``` -sudo make clean + +and add a new section at the end of the main `Dockerfile` in + ``` -* Build the new image +images/custom/Dockerfile +``` + +for your code immediately **before** the permissiosn section. + +### Build and Test your changes + +Build your modified container from the **root** `jupyteruser-sd2e` directory using the (hopefully) helpful make commands + ``` +sudo make clean sudo make build sd2e ``` -* Test the new image + +If your build succeeds, you can then test with + ``` sudo make test sd2e ``` -* Push your branch to your Github fork when you're ready for integration -* Submit a Pull Request against the SD2E `development` branch (template provided once again) -* Your PR will be reviewed: - * A Docker image will be built from your code and will be available in about an hour on the test Jupyter server: https://jupyter-staging.sd2e.org/ + +Visit your local jupyter environment using either the localhost URL if on your own computer, or the external URL if testing in the cloud. + +### Submit pull request + +When done testing, commit and push your changes to your personal fork of the repository. Then, you can create a pull request against the SD2E `development` branch. Once again, we provide a template to help populate the necessary information about your changes. + +After we review and accept your pull request: + * A Docker image will be built from your code and will be available in about an hour on the test Jupyter server + * https://jupyter-staging.sd2e.org/ * Both the SD2E development team and you must review and validate * The updated/new functionality * Completeness and correctness of provided documenation diff --git a/README.md b/README.md index 1b156dd..0884fd0 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,23 @@ ## What it Gives You -* Jupyter Notebook server (v4.0.x or v3.2.x, see tag) +* Jupyter Notebook \(v5.2.0\)server * Conda Python 3.4.x and Python 2.7.x environments -* pandas, matplotlib, scipy, seaborn, scikit-learn, numba pre-installed -* R-essentials (including an R kernel) -* Plotly, igraph, and networkx for graphs and plots +* Pre-installed python packages (highlights) + * pandas + * matplotlib 2.1.2 + * numpy, scipy + * seaborn + * scikit-learn + * numba + * pyemd +* R-kernel +* Plotly, igraph, networkx, graphviz for graphs and plots * MIT's [Open Probabilistic Programming Stack](http://probcomp.org/) * [Common Lisp kernel and stack](http://www.sbcl.org/) * Bioconda and Bioconductor * BioPython +* Git ### SD2E Specific Features diff --git a/images/base/Dockerfile b/images/base/Dockerfile index 2874eac..5c0e683 100644 --- a/images/base/Dockerfile +++ b/images/base/Dockerfile @@ -23,9 +23,10 @@ echo 'PS1="$JUPYTERHUB_USER@\h:\w\$ "' >> /home/jupyter/.bashrc && \ # allow users to add kernels chmod 777 /opt/conda/share/jupyter/kernels && \ # Environment variables -echo "WORK=/home/jupyter/tacc-work\n\ -SD2_DATA=/home/jupyter/sd2e-community\n\ -PYTHONUSERBASE=/home/jupyter/tacc-work/jupyter_packages\n" >> /home/jupyter/.bashrc +echo "export JPY_USER=$JUPYTERHUB_USER\n\ +export WORK=/home/jupyter/tacc-work\n\ +export SD2_DATA=/home/jupyter/sd2e-community\n\ +export PYTHONUSERBASE=/home/jupyter/tacc-work/jupyter_packages\n" >> /home/jupyter/.bashrc # Add readme ADD docs/SD2E_README.md /home/jupyter/SD2E_README.md @@ -52,8 +53,8 @@ cd /root && \ rm -rf sd2e-jupyter-ascending && \ # Ubuntu Packages apt-get update && \ - # jq is required for abaco CLI - apt-get install -y tree less jq rsync column && \ + # jq and bsdmainutils->column is required for abaco CLI + apt-get install -y tree less jq rsync bsdmainutils && \ #apt-get install -y tree less openjdk-8-jre ca-certificates-java && \ docker-clean # Replace stock Agave CLI with SD2E-customized tooling diff --git a/images/custom/Dockerfile b/images/custom/Dockerfile index a8667dd..63e0ce7 100644 --- a/images/custom/Dockerfile +++ b/images/custom/Dockerfile @@ -25,7 +25,6 @@ USER root LABEL "comment"="Support for Open Probabilistic Programming Stack" "issue"="SD2E/jupyteruser-sd2e/issues/2" # Libraries -ENV MPLBACKEND=Agg RUN apt-get update && \ apt-get install -y libboost-all-dev libgsl0-dev ccache && \ apt-get clean && \ @@ -37,7 +36,7 @@ conda install -n python2 --quiet --yes flask jsonschema patsy requests six && \ gcc_impl_linux-64 gcc_linux-64 \ gxx_impl_linux-64 gxx_linux-64 && \ # Install probcomp software stack -bash -c "source activate python2 && pip2 install git+https://github.com/probcomp/Venturecxx@v0.5.1.1 && \ +bash -c "export MPLBACKEND=Agg && source activate python2 && pip2 install git+https://github.com/probcomp/Venturecxx@v0.5.1.1 && \ pip2 install git+https://github.com/probcomp/cgpm.git@v0.1.2 && \ pip2 install git+https://github.com/probcomp/crosscat.git@v0.1.56.1 && \ pip2 install git+https://github.com/probcomp/bayeslite.git@v0.3.2 && \ From e6135307321d0dac105ce8fee955f8fd436afbce Mon Sep 17 00:00:00 2001 From: Greg Zynda Date: Wed, 14 Feb 2018 13:12:14 -0500 Subject: [PATCH 5/7] Fixed JPY_USER variable --- images/base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/base/Dockerfile b/images/base/Dockerfile index 5c0e683..d779ecc 100644 --- a/images/base/Dockerfile +++ b/images/base/Dockerfile @@ -23,7 +23,7 @@ echo 'PS1="$JUPYTERHUB_USER@\h:\w\$ "' >> /home/jupyter/.bashrc && \ # allow users to add kernels chmod 777 /opt/conda/share/jupyter/kernels && \ # Environment variables -echo "export JPY_USER=$JUPYTERHUB_USER\n\ +echo "export JPY_USER=\$JUPYTERHUB_USER\n\ export WORK=/home/jupyter/tacc-work\n\ export SD2_DATA=/home/jupyter/sd2e-community\n\ export PYTHONUSERBASE=/home/jupyter/tacc-work/jupyter_packages\n" >> /home/jupyter/.bashrc From fbdf8c8bc5bc9514158a73dcab9ce25628cdcbfd Mon Sep 17 00:00:00 2001 From: Greg Zynda Date: Thu, 15 Feb 2018 12:24:46 -0500 Subject: [PATCH 6/7] User installed packages are now discovered in python2 env --- images/base/Dockerfile | 4 ++++ images/base/src/activate-python.sh | 3 +++ 2 files changed, 7 insertions(+) create mode 100644 images/base/src/activate-python.sh diff --git a/images/base/Dockerfile b/images/base/Dockerfile index d779ecc..a8e81ad 100644 --- a/images/base/Dockerfile +++ b/images/base/Dockerfile @@ -27,6 +27,10 @@ echo "export JPY_USER=\$JUPYTERHUB_USER\n\ export WORK=/home/jupyter/tacc-work\n\ export SD2_DATA=/home/jupyter/sd2e-community\n\ export PYTHONUSERBASE=/home/jupyter/tacc-work/jupyter_packages\n" >> /home/jupyter/.bashrc +# Add activation sripts +ADD src/activate-python.sh /opt/conda/etc/conda/activate.d/activate-python.sh +ADD src/activate-python.sh /opt/conda/envs/python2/etc/conda/activate.d/activate-python.sh +RUN find /opt/conda -type d -name activate.d -exec chmod -R a+rx {} \; # Add readme ADD docs/SD2E_README.md /home/jupyter/SD2E_README.md diff --git a/images/base/src/activate-python.sh b/images/base/src/activate-python.sh new file mode 100644 index 0000000..58c18e9 --- /dev/null +++ b/images/base/src/activate-python.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +export PYTHONUSERBASE=/home/jupyter/tacc-work/jupyter_packages From 7c72f59f9d213931dc188d3387b17a057fb08eec Mon Sep 17 00:00:00 2001 From: Greg Zynda Date: Thu, 15 Feb 2018 14:25:32 -0600 Subject: [PATCH 7/7] Incremented Version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 0d91a54..9e11b32 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0 +0.3.1