feat: Update Dockerfile.jupyter and remove the workaround.

Dockerfile.jupyter

@@ -30,19 +30,15 @@ RUN if [ "$dev_mode" = "true" ]; then \
       cd sparkmagic && pip install -e . && cd ../ ; \
     else pip install sparkmagic ; fi
 
-# Jupyter extensions changed in >7.x.x
-# For now (workaround), let's pin to 6 to avoid breaking things
-# xref: https://github.com/jupyter-incubator/sparkmagic/issues/825
-RUN pip install notebook==6.5.5
 
 RUN mkdir /home/$NB_USER/.sparkmagic
 COPY sparkmagic/example_config.json /home/$NB_USER/.sparkmagic/config.json
 RUN sed -i 's/localhost/spark/g' /home/$NB_USER/.sparkmagic/config.json
-RUN jupyter nbextension enable --py --sys-prefix widgetsnbextension
+RUN pip install ipywidgets
 RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkkernel
 RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/pysparkkernel
 RUN jupyter-kernelspec install --user $(pip show sparkmagic | grep Location | cut -d" " -f2)/sparkmagic/kernels/sparkrkernel
-RUN jupyter serverextension enable --py sparkmagic
+RUN jupyter server extension enable --py sparkmagic
 
 USER root
 RUN chown $NB_USER /home/$NB_USER/.sparkmagic/config.json