Merge pull request #537 from Epistimio/release-v0.1.12rc

Release v0.1.12rc

.github/workflows/build.yml

@@ -130,6 +130,7 @@ jobs:
       if: startsWith(github.ref, 'refs/tags')
       uses: pypa/gh-action-pypi-publish@master
       with:
+        user: __token__
         password: ${{ secrets.pypi_password }}
   conda:
     needs: [test, backward-compatibility]

.github/workflows/release_master_dev_resync.yml

@@ -0,0 +1,38 @@
+on:
+  release:
+    types:
+      [published]
+
+jobs:
+  createPullRequest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Update version
+        run: |
+            # Get current version tag
+            echo "Version tag: $(git describe --tags)"
+            git describe --tags | cut -d 'v' -f 2 >> tests/functional/backward_compatibility/versions.txt
+      - name: Create Pull Request
+        id: cpr
+        uses: peter-evans/create-pull-request@v3
+        with:
+          commit-message: 'Update backward comp test versions'
+          branch: ci/sync_master_back_to_dev
+          base: master 
+          delete-branch: true
+          title: 'Merge back master in develop after release'
+          body: |
+            Auto-generated by [create-pull-request][1]
+
+            [1]: https://github.com/peter-evans/create-pull-request
+          labels: |
+            automated pr
+          assignees: bouthilx
+          reviewers: bouthilx
+
+      - name: Check outputs
+        run: |
+          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

.pylintrc

@@ -52,7 +52,7 @@ confidence=
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
 disable=abstract-class-instantiated,useless-super-delegation,no-member,keyword-arg-before-vararg,unidiomatic-typecheck,redefined-outer-name,fixme,F0401,intern-builtin,wrong-import-position,wrong-import-order,
-        C0415, F0010, R0205, R1705, R1711, R1720, W0106, W0107, W0127, W0706, C0330, C0326
+        C0415, F0010, R0205, R1705, R1711, R1720, W0106, W0107, W0127, W0706, C0330, C0326, W1203
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

docs/src/code/plotting.rst

@@ -2,5 +2,9 @@
 Plotting
 ********
 
-.. automodule:: orion.plotting
-   :members:
+.. toctree::
+   :maxdepth: 1
+   :caption: Modules
+
+   plotting/base
+   plotting/plotly

docs/src/code/plotting/base.rst

@@ -0,0 +1,5 @@
+Plotting Base Backend
+=====================
+
+.. automodule:: orion.plotting.base
+   :members:

docs/src/code/plotting/plotly.rst

@@ -0,0 +1,5 @@
+Plotly backend
+==============
+
+.. automodule:: orion.plotting.backend_plotly
+   :members:

docs/src/developer/release.rst

@@ -18,13 +18,18 @@ such as the README.rst.
    place in the *develop* branch. More information is available in :ref:`standard-vcs`.
 #. In README.rst, replace any link pointing to ``https://orion.readthedocs.io/en/latest/**`` to
    ``https://orion.readthedocs.io/en/stable/**``.
-#. Create a new pull request for the branch created in the last step and list all the changes by
-   category. Example: https://github.com/Epistimio/orion/pull/283.
 #. Update the **Citation** section in the project's README.rst with the latest version of Oríon.
 #. Update the ``ROADMAP.md``.
 #. Update the linters ``black``, ``isort``, ``pylint``, and ``doc8`` to their latest versions in
    ``tox.ini``, and address any new error.
 #. Run the stress tests according to the instructions in stress test's documentation.
+#. Create a new pull request for the branch created in the first step. The pull request should be
+   using the base `master` instead of `develop`.
+#. Go to the `release page`_ copy paste the current draft
+   (which was automatically wrote by the `release drafter`_ app) into the description of the new
+   pull request. Adapt if necessary, sometimes new features are spread across multiple pull requests
+   or some pull requests are changes that should not figure in the release description, like
+   merging master back to develop branch (ex: `PR #510 <https://github.com/Epistimio/orion/pull/510>`_).
 
 .. _release-make:
 
@@ -34,17 +39,19 @@ Once the release is thoroughly tested and the core contributors are confident in
 time to create the release artifacts and publish the release.
 
 #. Merge the release candidate branch to master (no fast-forward merge, we want a merge commit).
-#. Create a `new draft release <https://github.com/Epistimio/orion/releases/new>` on GitHub. Set the
-   target branch to *master* and the tag version to ``v{version}``. Reuse the changelog from the
-   release candidate pull request's for the description. See the `0.1.6
-   <https://github.com/Epistimio/orion/releases/tag/v0.1.6>`_ version example.
-#. Merge the master branch back to develop.
 #. Delete the release candidate branch.
-#. Update the backward compability tests by adding the new version in develop branch
-   and make a pull request on develop.
+#. Go back to the `release page`_, edit the title to describe important changes and publish the
+   release. The version should already be updated automatically by `release drafter`_. Adjust if
+   necessary.
+#. The publication should trigger a github action which will update the backward compatibility tests
+   and create a pull request to merge the master back to develop branch. Wait for tests to pass and
+   merge.
 
 Once the release is made, the :ref:`ci` will be automatically started by Github. The code will
-then be published on PyPI_ and Anaconda_ automatically if the tests passes.
+then be published on PyPI_ and Anaconda_ automatically if the tests passes. If test fails for
+random reasons (sometimes mongodb setup fails during build), you can fetch the tagged version
+locally and publish on PyPI_ and Anaconda_ manually, using ``$ tox -e release`` and
+``./conda/upload.sh``.
 
 After the release
 =================
@@ -60,6 +67,8 @@ version exists so they can update their version or learn about Oríon.
 * Announce the new release on relevant communication channels (e.g., email, forums, google groups)
 * Congratulations, you published a new version of Oríon!
 
+.. _release drafter: https://github.com/marketplace/actions/release-drafter
+.. _release page: https://github.com/Epistimio/orion/releases
 .. _GitHub: https://github.com/Epistimio/orion/releases
 .. _Zenodo: https://doi.org/10.5281/zenodo.3478592
 .. _PyPI: https://pypi.org/project/orion/

docs/src/user/config.rst

@@ -119,6 +119,7 @@ Full Example of Global Configuration
         cli_change_type: break
         code_change_type: break
         config_change_type: break
+        orion_version_change: False
         ignore_code_changes: False
         manual_resolution: False
         non_monitored_arguments: []
@@ -482,6 +483,7 @@ Experiment Version Control
         cli_change_type: break
         code_change_type: break
         config_change_type: break
+        orion_version_change: False
         ignore_code_changes: False
         manual_resolution: False
         non_monitored_arguments: []
@@ -618,3 +620,17 @@ config_change_type
     incompatible results. The child cannot access the trials from parent if ``config_change_type``
     is ``break``.  The parent cannot access trials from child if ``config_change_type`` is
     ``unsure`` or ``break``.
+
+
+.. _config_evc_orion_version_change:
+
+orion_version_change
+~~~~~~~~~~~~~~~~~~~~
+
+:Type: bool
+:Default: False
+:Env var: ORION_EVC_ORION_VERSION_CHANGE
+:Description:
+    If ``True``, set orion version change as resolved if branching event occured.
+    Child and parent experiment have access to all trials from each other
+    when the only difference between them is the orion version used during execution.

docs/src/user/evc.rst

@@ -166,6 +166,7 @@ Source of conflicts
 3. Script configuration file modification
 4. Optimization space modification (new hyper-parameters or change of prior distribution)
 5. Algorithm configuration modification
+6. Orion version change
 
 Iterative Results
 =================

docs/src/user/library/evc_results.rst

@@ -14,9 +14,9 @@ trials of the specific experiment.
 
    import pprint
 
-   from orion.client import create_experiment
+   from orion.client import get_experiment
 
-   experiment = create_experiment(name="orion-tutorial-with-momentum")
+   experiment = get_experiment(name="orion-tutorial-with-momentum")
 
    print(experiment.name)
    pprint.pprint(experiment.stats)

docs/src/user/library/results.rst

@@ -4,21 +4,21 @@ Results
 You can fetch experiments and trials using python code. There is no need to understand the
 specific database backend used (such as MongoDB) since you can fetch results using the
 :class:`orion.client.experiment.ExperimentClient` object.
-The helper function :py:func:`orion.client.create_experiment`
+The helper function :py:func:`orion.client.get_experiment`
 provides a simple way to fetch experiments
 using their unique names. You do not need to explicitly open a connection to the database since it
 will automatically infer its configuration from the global configuration file as when calling Oríon
 in commandline. Otherwise you can specify the configuration directly to
-:py:func:`create_experiment() <orion.client.create_experiment>`. Take a look at the documentation
+:py:func:`get_experiment() <orion.client.get_experiment>`. Take a look at the documentation
 for more details on all configuration arguments that are supported.
 
 .. code-block:: python
 
    # Database automatically inferred
-   create_experiment(name="orion-tutorial")
+   get_experiment(name="orion-tutorial")
 
    # Database manually set
-   create_experiment(
+   get_experiment(
        name="orion-tutorial",
        storage={
            'type': 'legacy',
@@ -33,9 +33,9 @@ For a complete example, here is how you can fetch trials from a given experiment
 
    import pprint
 
-   from orion.client import create_experiment
+   from orion.client import get_experiment
 
-   experiment = create_experiment(name="orion-tutorial")
+   experiment = get_experiment(name="orion-tutorial")
 
    pprint.pprint(experiment.stats)
 

docs/src/user/storage.rst

@@ -159,15 +159,20 @@ ExperimentClient
 ~~~~~~~~~~~~~~~~
 
 The experiment client must be created with the helper function
-:py:func:`create_experiment() <orion.client.create_experiment>` which will take care of
-initiating the storage backend and create a new experiment if non-existant or simply load
-the corresponding experiment from the storage.
+:py:func:`get_experiment() <orion.client.get_experiment>` which will take care of
+initiating the storage backend and load the corresponding experiment from the storage.
+To create a new experiment use :py:func:`create_experiment() <orion.client.create_experiment>`.
 
-There is a small subset of methods to fetch trials or create new ones. We focus here
+There is a small subset of methods to fetch trials or register new ones. We focus here
 on the methods for loading or creation of trials in particular, see
 :py:class:`ExperimentClient <orion.client.experiment.ExperimentClient>` for documentation
 of all methods.
 
+The experiment client can be loaded in read-only or read/write mode. Make sure to
+load the experiment with the proper mode if you want to edit the database.
+For full read/write/execution rights, use
+:py:func:`create_experiment() <orion.client.create_experiment>`.
+
 Here is a short example to fetch trials or insert a new one.
 
 .. code-block:: python

src/orion/algo/asha.py

@@ -119,6 +119,7 @@ def __init__(
         )
 
         self.trial_info = {}  # Stores Trial -> Bracket
+        self.sampled = set()
 
         try:
             fidelity_index = self.fidelity_index
@@ -197,7 +198,7 @@ def seed_rng(self, seed):
     @property
     def state_dict(self):
         """Return a state dict that can be used to reset the state of the algorithm."""
-        return {"rng_state": self.rng.get_state()}
+        return {"rng_state": self.rng.get_state(), "sampled": self.sampled}
 
     def set_state(self, state_dict):
         """Reset the state of the algorithm based on the given state_dict
@@ -206,6 +207,7 @@ def set_state(self, state_dict):
         """
         self.seed_rng(0)
         self.rng.set_state(state_dict["rng_state"])
+        self.sampled = state_dict["sampled"]
 
     def suggest(self, num=1):
         """Suggest a `num` of new sets of parameters.
@@ -226,6 +228,7 @@ def suggest(self, num=1):
 
             if candidate:
                 logger.debug("Promoting")
+                self.sampled.add(self.get_id(candidate, ignore_fidelity=False))
                 return [candidate]
 
         point = self._grow_point_for_bottom_rung()
@@ -247,6 +250,7 @@ def suggest(self, num=1):
 
         logger.debug("Sampling for bracket %s %s", idx, self.brackets[idx])
 
+        self.sampled.add(self.get_id(point, ignore_fidelity=False))
         return [tuple(point)]
 
     def _grow_point_for_bottom_rung(self):
@@ -284,13 +288,15 @@ def _grow_point_for_bottom_rung(self):
 
         return point
 
-    def get_id(self, point):
-        """Compute a unique hash for a point based on params, but not fidelity level."""
+    def get_id(self, point, ignore_fidelity=True):
+        """Compute a unique hash for a point based on params, without fidelity level by default."""
         _point = list(point)
-        non_fidelity_dims = _point[0 : self.fidelity_index]
-        non_fidelity_dims.extend(_point[self.fidelity_index + 1 :])
+        if ignore_fidelity:
+            non_fidelity_dims = _point[0 : self.fidelity_index]
+            non_fidelity_dims.extend(_point[self.fidelity_index + 1 :])
+            _point = non_fidelity_dims
 
-        return hashlib.md5(str(non_fidelity_dims).encode("utf-8")).hexdigest()
+        return hashlib.md5(str(_point).encode("utf-8")).hexdigest()
 
     def observe(self, points, results):
         """Observe evaluation `results` corresponding to list of `points` in
@@ -300,7 +306,16 @@ def observe(self, points, results):
         """
         for point, result in zip(points, results):
 
+            full_id = self.get_id(point, ignore_fidelity=False)
+            if full_id not in self.sampled:
+                logger.info(
+                    "Ignoring point %s because it was not sampled by current algo.",
+                    full_id,
+                )
+                continue
+
             _id = self.get_id(point)
+
             bracket = self.trial_info.get(_id)
 
             if not bracket:
@@ -316,6 +331,7 @@ def observe(self, points, results):
                             _id, fidelity
                         )
                     )
+
                 bracket = brackets[0]
 
             try:

src/orion/algo/hyperband.py

@@ -126,6 +126,7 @@ def __init__(self, space, seed=None, repetitions=numpy.inf):
         self.trial_info_wo_fidelity = (
             {}
         )  # Stores Point id (with no fidelity) -> Bracket
+        self.sampled = set()
 
         self.points_in_suggest_call = {}
 
@@ -211,6 +212,7 @@ def state_dict(self):
             "rng_state": self.rng.get_state(),
             "seed": self.seed,
             "executed_times": self.executed_times,
+            "sampled": self.sampled,
         }
 
     def set_state(self, state_dict):
@@ -221,6 +223,7 @@ def set_state(self, state_dict):
         self.seed_rng(state_dict["seed"])
         self.rng.set_state(state_dict["rng_state"])
         self.executed_times = state_dict["executed_times"]
+        self.sampled = state_dict["sampled"]
 
     def suggest(self, num=1):
         """Suggest a number of new sets of parameters.
@@ -250,6 +253,8 @@ def suggest(self, num=1):
                 samples += bracket.sample()
 
         if samples:
+            for sample in samples:
+                self.sampled.add(self.get_id(sample, ignore_fidelity=False))
             return samples
 
         # All brackets are filled
@@ -259,6 +264,8 @@ def suggest(self, num=1):
                 samples += bracket.promote()
 
         if samples:
+            for sample in samples:
+                self.sampled.add(self.get_id(sample, ignore_fidelity=False))
             return samples
 
         # Either all brackets are done or none are ready and algo needs to wait for some trials to
@@ -349,6 +356,14 @@ def observe(self, points, results):
         """
         for point, result in zip(points, results):
 
+            full_id = self.get_id(point, ignore_fidelity=False)
+            if full_id not in self.sampled:
+                logger.info(
+                    "Ignoring point %s because it was not sampled by current algo.",
+                    full_id,
+                )
+                continue
+
             bracket = self._get_bracket(point)
 
             try: