diff --git a/.dockerignore b/.dockerignore index 1b02a1889..5945e596e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -25,4 +25,3 @@ notebooks examples docs build -coverage.xml diff --git a/.gcloudignore b/.gcloudignore deleted file mode 100644 index 81a9704e5..000000000 --- a/.gcloudignore +++ /dev/null @@ -1,24 +0,0 @@ -.idea -*venv - -# See note in .dockerignore about the git folder. -.git -.github - -*.md -!README*.md - -logs/ -**/.ipynb_checkpoints -**/.pytest_cache -**/.eggs -**/*.pdf -**/*.log -**/*.egg-info -**/*.pyc -**/__pycache__ -notebooks -examples -docs -build -coverage.xml diff --git a/.github/workflows/pythontest.yaml b/.github/workflows/pythontest.yaml index a9f5d918c..1b98c7cee 100644 --- a/.github/workflows/pythontest.yaml +++ b/.github/workflows/pythontest.yaml @@ -29,12 +29,14 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v2 - - name: Build and test panoptes-utils in container + - name: Build panoptes-utils in container run: | docker build -t panoptes-utils:testing -f tests/Dockerfile . mkdir -p logs && chmod -R 777 logs mkdir -p build && chmod -R 777 build - docker run --rm -i -v "${PWD}/build:/var/panoptes/panoptes-utils/build" -v "${PWD}/logs:/var/panoptes/logs" panoptes-utils:testing + - name: Test panoptes-utils + run: | + docker run --rm -i -v "${PWD}/build:/app/build" -v "${PWD}/logs:/app/logs" panoptes-utils:testing - name: Upload coverage report to codecov.io uses: codecov/codecov-action@v1 if: success() diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 622b4677d..ac8fce8f6 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,38 @@ Changelog ========= +0.2.31 - 2020-01-31 +------------------- + +Added +^^^^^ + +* Docker musical chairs: + + * Add Dockerfile to be ``panoptes-utils`` but don't use for testing. (#264) + * Generic ``panoptes-utils`` with ``panoptes-config-server`` as default command example. (#264) + * Move conda ``environment`` file into ``docker`` folder. (#264) + +Bugs Fixed +^^^^^^^^^^ + +* ``parse_config_directories`` no longer modifies dictionary in place. (#264) + +Changed +^^^^^^^ + +* Clean out the Contributing guide to point to POCS. (#264) +* Removing ``pillow<7`` requirement. (#264) + +Removed +^^^^^^^ + +* Removing unused new cli experiment. (#264) +* Removed all ``PANDIR`` and ``PANLOG`` references. Closes #263. (#264) +* Removed ``astroplan`` from dependencies. (#264) + + + 0.2.30 - 2021-01-14 ------------------- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6af8dd6b5..8798cf9a1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,146 +1,4 @@ -Please see the -[code of conduct](https://github.com/panoptes/POCS/blob/develop/CODE_OF_CONDUCT.md) -for our playground rules and follow them during all your contributions. +# Contributing -# Getting Started - -We prefer that all changes to POCS have an associated -[GitHub Issue in the project](https://github.com/panoptes/POCS/issues) -that explains why it is needed. This allows us to debate the best -approach to address the issue before folks spend a lot of time -writing code. If you are unsure about a possible contribution to -the project, please contact the project owners about your idea; -of course, an [issue](https://github.com/panoptes/POCS/issues) is a -good way to do this. - -# Pull Request Process -_This is a summary of the process. See -[the POCS wiki](https://github.com/panoptes/POCS/wiki/PANOPTES-Feature-Development-Process) -for more info._ - -* Pre-requisites - - Ensure you have a [github account.](https://github.com/join) - - If the change you wish to make is not already an - [Issue in the project](https://github.com/panoptes/POCS/issues), - please create one specifying the need. -* Process - - Create a fork of the repository and use a topic branch within your fork to make changes. - - All of our repositories have a default branch of `develop` when you first clone them, but - your work should be in a separate branch. - - Create a branch with a descriptive name, e.g.: - - `git checkout -b new-camera-simulator` - - `git checkout -b issue-28` - - Ensure that your code meets this project's standards (see Testing and Code Formatting below). - - Run `python setup.py test` from the `$POCS` directory before pushing to github - - Squash your commits so they only reflect meaningful changes. - - Submit a pull request to the repository, be sure to reference the issue number it - addresses. - - -# Setting up Local Environment - - Follow instructions in the [README](https://github.com/panoptes/POCS/blob/develop/README.md) - as well as the [Coding in PANOPTES](https://github.com/panoptes/POCS/wiki/Coding-in-PANOPTES) - document. - - -# Testing - - All changes should have corresponding tests and existing tests should pass after - your changes. - - For more on testing see the - [Coding in PANOPTES](https://github.com/panoptes/POCS/wiki/Coding-in-PANOPTES) page. - -# Code Formatting - -- All Python should use [PEP 8 Standards](https://www.python.org/dev/peps/pep-0008/) - - Line length is set at 100 characters instead of 80. - - It is recommended to have your editor auto-format code whenever you save a file - rather than attempt to go back and change an entire file all at once. - - You can also use - [yapf (Yet Another Python Formatter)](https://github.com/google/yapf) - for which POCS includes a style file (.style.yapf). For example: - ```bash - # cd to the root of your workspace. - cd $(git rev-parse --show-toplevel) - # Format the modified python files in your workspace. - yapf -i $(git diff --name-only | egrep '\.py$') - ``` -- Do not leave in commented-out code or unnecessary whitespace. -- Variable/function/class and file names should be meaningful and descriptive. -- File names should be lower case and underscored, not contain spaces. For example, `my_file.py` -instead of `My File.py`. -- Define any project specific terminology or abbreviations you use in the file you use them. -- Use root-relative imports (i.e. relative to the POCS directory). This means that rather - than using a directory relative imports such as: - ```python - from panoptes.utils.base import PanBase - from panoptes.utils.time import current_time - ``` - Import from the top-down instead: - ```python - from pocs.base import PanBase - from panoptes.utils.time import current_time - ``` - The same applies to code inside of `peas`. -- Test imports are slightly different because `pocs/tests` and `peas/tests` are not Python - packages (those directories don't contain an `__init__.py` file). For imports of `pocs` or - `peas` code, use root-relative imports as described above. For importing test packages and - modules, assume the test doing the imports is in the root directory. - -# Log Messages - -Use appropriate logging: -- Log level: - - DEBUG (i.e. `self.logger.debug()`) should attempt to capture all run-time - information. - - INFO (i.e. `self.logger.info()`) should be used sparingly and meant to convey - information to a person actively watching a running unit. - - WARNING (i.e. `self.logger.warning()`) should alert when something does not - go as expected but operation of unit can continue. - - ERROR (i.e. `self.logger.error()`) should be used at critical levels when - operation cannot continue. -- The logger supports variable information without the use of the `format` method. -- There is a `say` method available on the main `POCS` class that is meant to be -used in friendly manner to convey information to a user. This should be used only -for personable output and is typically displayed in the "chat box"of the PAWS -website. These messages are also sent to the INFO level logger. - -#### Logging examples: - -_Note: These are meant to illustrate the logging calls and are not necessarily indicative of real -operation_ - -```py -self.logger.info("PANOPTES unit initialized: {}", self.config['name']) - -self.say("I'm all ready to go, first checking the weather") - -self.logger.debug("Setting up weather station") - -self.logger.warning('Problem getting wind safety: {}'.format(e)) - -self.logger.debug("Rain: {} Clouds: {} Dark: {} Temp: {:.02f}", - is_raining, - is_cloudy, - is_dark, - temp_celsius -) - -self.logger.error('Unable to connect to AAG Cloud Sensor, cannot continue') -``` - -#### Viewing log files - -- You typically want to follow an active log file by using `tail -F` on the command line. -- The [`grc`](https://github.com/garabik/grc) (generic colouriser) can be used with -`tail` to get pretty log files. - -``` -(panoptes-env) $ grc tail -F $PANDIR/logs/pocs_shell.log -``` - -The following screenshot shows commands entered into a `jupyter-console` in the top -panel and the log file in the bottom panel. - -

- -

+See [the POCS wiki](https://github.com/panoptes/POCS/wiki/PANOPTES-Feature-Development-Process) +for more info on how to contribute to the various PANOPTES repositories. diff --git a/README.md b/README.md index 92a2ff275..243b6b9ee 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,36 @@ To install type: pip install panoptes-utils ``` -Full options for local install: +Full options for install: ```bash pip install -e ".[config,docs,images,testing,social]" ``` See the full documentation at: https://panoptes-utils.readthedocs.io + +Docker Service +============== + +The `docker` folder defines an image that can be used as the base for other +PANOPTES services. + +The `Dockerfile` is built by the `cloudbuild.yaml` and stored in Google +Registry as `gcr.io/panoptes-exp/panoptes-utils:latest`. + +You can pull the image like any other docker image: + +``` +docker pull gcr.io/panoptes-exp/panoptes-utils:latest +``` + +Config Server +------------- + +There is also a service defined in `docker-compose.yaml` that will run the +`panoptes-config-server` cli tool. + +```bash +PANOPTES_CONFIG_FILE=/path/to/config.yaml docker-compose \ + -f docker/docker-compose.yaml up +``` diff --git a/conftest.py b/conftest.py index 4ae300d48..6a3d2b3ba 100644 --- a/conftest.py +++ b/conftest.py @@ -25,8 +25,7 @@ "{message}" # Put the log file in the tmp dir. -log_dir = os.getenv('PANLOG', '/var/panoptes/logs') -log_file_path = os.path.realpath(f'{log_dir}/panoptes-testing.log') +log_file_path = os.path.realpath(f'logs/panoptes-testing.log') startup_message = f' STARTING NEW PYTEST RUN - LOGS: {log_file_path} ' logger.add(log_file_path, enqueue=True, # multiprocessing @@ -75,7 +74,7 @@ def pytest_addoption(parser): @pytest.fixture(scope='session') def config_path(): - return os.getenv('PANOPTES_CONFIG_FILE', '/var/panoptes/panoptes-utils/tests/testing.yaml') + return os.getenv('PANOPTES_CONFIG_FILE', 'tests/testing.yaml') @pytest.fixture(scope='function', params=_all_databases) @@ -106,7 +105,7 @@ def save_environ(): @pytest.fixture(scope='session') def data_dir(): - return os.path.expandvars('/var/panoptes/panoptes-utils/tests/data') + return os.path.expandvars('tests/data') @pytest.fixture(scope='function') diff --git a/docker/.gcloudignore b/docker/.gcloudignore new file mode 100644 index 000000000..81cd1a57e --- /dev/null +++ b/docker/.gcloudignore @@ -0,0 +1,2 @@ +cloudbuild.yaml +docker-compose.yaml diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..e6bf51df1 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,70 @@ +ARG image_url=ubuntu +ARG image_tag=latest +FROM ${image_url}:${image_tag} AS pocs-utils + +LABEL description="Installs the dependencies for panoptes-utils." +LABEL maintainers="developers@projectpanoptes.org" +LABEL repo="github.com/panoptes/panoptes-utils" + +ENV DEBIAN_FRONTEND=noninteractive +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 + +ARG panuser=pocs-user +ARG userid=1000 +ARG pip_install_extras="[config]" + +ENV PANUSER $panuser +ENV USERID $userid + +# Install system dependencies. +RUN echo "Building from ${image_name}:${image_tag}" && \ + apt-get update && apt-get install --no-install-recommends --yes \ + bzip2 ca-certificates \ + wget gcc git pkg-config sudo less udev wait-for-it \ + dcraw exiftool \ + astrometry.net \ + libcfitsio-dev libcfitsio-bin \ + libfreetype6-dev libpng-dev libjpeg-dev libffi-dev && \ + useradd -u ${USERID} -o -c "Captain POCS" \ + -p panoptes -m -G plugdev,dialout,users,sudo ${PANUSER} && \ + # Allow sudo without password. + echo "%sudo ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \ + # Setup SSH so localhost works without password + mkdir -p "/home/${panuser}/.ssh" && \ + echo "Host localhost\n\tStrictHostKeyChecking no\n" >> "/home/${panuser}/.ssh/config" + +USER "${userid}" + +# Miniconda +WORKDIR /tmp +RUN echo "Installing conda via miniforge" && \ + sudo mkdir -p /conda && \ + sudo chown -R "${PANUSER}:${PANUSER}" /conda && \ + # Miniforge + wget "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$(uname -m).sh" \ + -O install-miniforge.sh && \ + /bin/sh install-miniforge.sh -b -f -p /conda && \ + # Initialize conda for the shells. + /conda/bin/conda init bash + +ENV PATH "/home/${PANUSER}/.local/bin:$PATH" + +COPY docker/environment.yaml . +RUN /conda/bin/conda env update -n base -f environment.yaml + +RUN echo "Installing panoptes-pocs module with ${pip_install_extras}" && \ + /conda/bin/pip install "panoptes-utils${pip_install_extras}" && \ + # Cleanup + /conda/bin/pip cache purge && \ + /conda/bin/conda clean -tipy && \ + sudo apt-get autoremove --purge --yes && \ + sudo apt-get autoclean --yes && \ + sudo apt-get --yes clean && \ + sudo rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY docker/docker-compose.yaml . + +# We are still the PANUSER. +ENTRYPOINT [ "/usr/bin/env", "bash", "-ic" ] +CMD [ "panoptes-config-server", "--help" ] diff --git a/docker/cloudbuild.yaml b/docker/cloudbuild.yaml new file mode 100644 index 000000000..4070276b6 --- /dev/null +++ b/docker/cloudbuild.yaml @@ -0,0 +1,49 @@ +options: + substitutionOption: "ALLOW_LOOSE" +timeout: 18000s # 5 hours + + +substitutions: + _PLATFORM: linux/arm64,linux/amd64 + +steps: + # Set up multiarch support + - name: "gcr.io/cloud-builders/docker" + id: "setup-buildx" + env: + - "DOCKER_CLI_EXPERIMENTAL=enabled" + args: + - "run" + - "--privileged" + - "--rm" + - "docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64" + waitFor: [ "-" ] + + # Build builder + - name: "gcr.io/cloud-builders/docker" + id: "build-builder" + env: + - "DOCKER_CLI_EXPERIMENTAL=enabled" + args: + - "buildx" + - "create" + - "--name=build" + - "--use" + - "--driver=docker-container" + waitFor: [ "setup-buildx" ] + + # Build the image. + - name: "gcr.io/cloud-builders/docker" + id: "build-images" + env: + - "DOCKER_CLI_EXPERIMENTAL=enabled" + args: + - "buildx" + - "build" + - "--platform=${_PLATFORM}" + - "-f=docker/Dockerfile" + - "--tag=gcr.io/${PROJECT_ID}/panoptes-utils:${TAG_NAME}" + - "--cache-from=gcr.io/${PROJECT_ID}/panoptes-utils:${TAG_NAME}" + - "--push" + - "." + waitFor: [ "build-builder" ] diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 000000000..8741a7c8e --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,26 @@ +version: '3.7' +services: + config-server: + image: gcr.io/panoptes-exp/panoptes-utils:latest + build: + context: . + dockerfile: ./Dockerfile + deploy: + mode: global + init: true + tty: true + container_name: config-server + hostname: config-server + network_mode: host + configs: + - config_file + environment: + PANOPTES_CONFIG_HOST: 0.0.0.0 + PANOPTES_CONFIG_PORT: 6563 + PANOPTES_CONFIG_FILE: + restart: on-failure + command: [ "panoptes-config-server --verbose run --config-file /app/config.yaml" ] + volumes: + - type: bind + source: "${PANOPTES_CONFIG_FILE}" + target: /app/config.yaml diff --git a/environment.yaml b/docker/environment.yaml similarity index 59% rename from environment.yaml rename to docker/environment.yaml index 86c4b1f63..dff8120b6 100644 --- a/environment.yaml +++ b/docker/environment.yaml @@ -3,25 +3,18 @@ channels: dependencies: - Flask - PyYAML - - astroplan - astropy - click - click-spinner - colorama - - coverage - gevent - loguru - matplotlib-base - numpy - - pandas - - photutils - - pillow<7 # https://github.com/ipython/ipython/issues/8052 + - pillow + - pip - python-dateutil - - python-dotenv - - readline - requests - ruamel - scipy - - sphinx_rtd_theme - - tweepy - typer diff --git a/setup.cfg b/setup.cfg index 7af78e7e0..9cb6dc470 100644 --- a/setup.cfg +++ b/setup.cfg @@ -65,7 +65,6 @@ exclude = config = Flask PyYAML - astroplan colorama gevent python-dateutil @@ -77,8 +76,8 @@ docs = images = matplotlib photutils + pillow scipy - pillow<7 # https://github.com/ipython/ipython/issues/8052 testing = coverage docker diff --git a/src/panoptes/utils/cli/__init__.py b/src/panoptes/utils/cli/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/panoptes/utils/cli/main.py b/src/panoptes/utils/cli/main.py deleted file mode 100644 index d2f4157fb..000000000 --- a/src/panoptes/utils/cli/main.py +++ /dev/null @@ -1,9 +0,0 @@ -import typer - -from panoptes.utils.cli import tests - -app = typer.Typer() -app.add_typer(tests.app, name='tests') - -if __name__ == "__main__": - app() diff --git a/src/panoptes/utils/cli/tests.py b/src/panoptes/utils/cli/tests.py deleted file mode 100644 index 7f1f34dcd..000000000 --- a/src/panoptes/utils/cli/tests.py +++ /dev/null @@ -1,58 +0,0 @@ -import os - -import click_spinner -import docker -import docker.errors -import typer - -app = typer.Typer() - - -@app.command() -def run( - directory: str = typer.Argument('/var/panoptes/panoptes-utils/'), - image_tag: str = typer.Argument('panoptes-utils:testing'), - log_dir: str = typer.Option('logs', - help='Location to stir log files, relative to project root.') -): - """Run the test suite.""" - client = docker.from_env() - - typer.secho(f'Log files will be output to {log_dir}') - os.makedirs(log_dir, exist_ok=True) - mount_volumes = { - os.path.realpath(log_dir): {'bind': '/var/panoptes/logs', 'mode': 'rw'}, - os.path.realpath('.'): {'bind': '/var/panoptes/panoptes-utils', 'mode': 'rw'} - } - - try: - client.images.get(image_tag) - except docker.errors.ImageNotFound: - typer.secho('Building test image (may take a few minutes)', fg=typer.colors.RED) - with click_spinner.spinner(): - build_test_image(directory, image_tag, docker_client=client) - - typer.echo(f'Starting testing on docker containers from {directory} with {image_tag}') - container = client.containers.run(image_tag, - name='panoptes-utils-testing', - detach=True, - auto_remove=True, - volumes=mount_volumes, - ) - - for line in container.logs(stream=True, follow=True): - typer.secho(line.decode(), nl=False) - - -def build_test_image(context_dir, image_tag, dockerfile=None, docker_client=None): - """Builds the docker image used for testing.""" - docker_client = docker_client or docker.from_env() - dockerfile = dockerfile or f'{context_dir}/tests/Dockerfile' - - test_image, build_logs = docker_client.images.build( - path=context_dir, - tag=image_tag, - dockerfile=dockerfile - ) - - return test_image diff --git a/src/panoptes/utils/config/helpers.py b/src/panoptes/utils/config/helpers.py index 0fce071d5..8b7cbd376 100644 --- a/src/panoptes/utils/config/helpers.py +++ b/src/panoptes/utils/config/helpers.py @@ -16,7 +16,7 @@ def load_config(config_files=None, parse=True, load_local=True): be via a running config server. This function supports loading of a number of different files. If no options - are passed to ``config_files`` then the default ``$PANDIR/conf_files/pocs.yaml`` + are passed to ``config_files`` then the default ``$PANOPTES_CONFIG_FILE`` will be loaded. ``config_files`` is a list and loaded in order, so the second entry will overwrite @@ -141,14 +141,14 @@ def parse_config_directories(directories, must_exist=False): .. doctest:: - >>> dirs_config = dict(base='/var/panoptes', foo='bar', baz='bam') + >>> dirs_config = dict(base='/tmp', foo='bar', baz='bam') >>> # If the relative dir doesn't exist but is required, return as is. >>> parse_config_directories(dirs_config, must_exist=True) - {'base': '/var/panoptes', 'foo': 'bar', 'baz': 'bam'} + {'base': '/tmp', 'foo': 'bar', 'baz': 'bam'} >>> # Default is to return anyway. >>> parse_config_directories(dirs_config) - {'base': '/var/panoptes', 'foo': '/var/panoptes/bar', 'baz': '/var/panoptes/bam'} + {'base': '/tmp', 'foo': '/tmp/bar', 'baz': '/tmp/bam'} >>> # If 'base' is not a valid absolute directory, return all as is. >>> dirs_config = dict(base='panoptes', foo='bar', baz='bam') @@ -164,14 +164,15 @@ def parse_config_directories(directories, must_exist=False): Returns: dict: The same directory but with relative directories resolved. """ + resolved_dirs = directories.copy() # Try to get the base directory first. - base_dir = directories.get('base', os.environ['PANDIR']) + base_dir = resolved_dirs.get('base', '.') if os.path.isdir(base_dir): logger.trace(f'Using base_dir={base_dir!r} for setting config directories') # Add the base directory to any relative dir. - for dir_name, rel_dir in directories.items(): + for dir_name, rel_dir in resolved_dirs.items(): # Only want relative directories. if rel_dir.startswith('/') is False: abs_dir = os.path.join(base_dir, rel_dir) @@ -183,9 +184,9 @@ def parse_config_directories(directories, must_exist=False): f'must_exist={must_exist!r} but abs_dir={abs_dir!r} does not exist, skipping') else: logger.trace(f'Setting {dir_name} to {abs_dir}') - directories[dir_name] = abs_dir + resolved_dirs[dir_name] = abs_dir - return directories + return resolved_dirs def _add_to_conf(config, conf_fn, parse=False): diff --git a/src/panoptes/utils/database/file.py b/src/panoptes/utils/database/file.py index b441aee2a..d69322ac0 100644 --- a/src/panoptes/utils/database/file.py +++ b/src/panoptes/utils/database/file.py @@ -22,9 +22,9 @@ def __init__(self, db_name='panoptes', storage_dir='json_store', **kwargs): Args: db_name (str, optional): Name of the database containing the collections. - storage_dir (str, optional): The name of the directory in $PANDIR where - the database files will be stored. Default is `json_store` for backwards - compatibility. + storage_dir (str, optional): The name of the directory where the + database files will be stored. Default is `json_store` in current + directory. Pass an absolute path for non-relative. """ super().__init__(db_name=db_name, **kwargs) @@ -32,7 +32,7 @@ def __init__(self, db_name='panoptes', storage_dir='json_store', **kwargs): self.db_folder = db_name # Set up storage directory. - self.storage_dir = os.path.join(os.environ['PANDIR'], storage_dir, self.db_folder) + self.storage_dir = os.path.join(storage_dir, self.db_folder) os.makedirs(self.storage_dir, exist_ok=True) def insert_current(self, collection, obj, store_permanently=True): @@ -45,7 +45,8 @@ def insert_current(self, collection, obj, store_permanently=True): # Overwrite current collection file with obj. to_json(obj, filename=current_fn, append=False) except Exception as e: - raise error.InvalidSerialization(f"Problem serializing object for insertion: {e} {current_fn} {obj!r}") + raise error.InvalidSerialization( + f"Problem serializing object for insertion: {e} {current_fn} {obj!r}") if not store_permanently: return result @@ -61,7 +62,8 @@ def insert(self, collection, obj): to_json(obj, filename=collection_fn) return obj_id except Exception as e: - raise error.InvalidSerialization(f"Problem inserting object into collection: {e}, {obj!r}") + raise error.InvalidSerialization( + f"Problem inserting object into collection: {e}, {obj!r}") def get_current(self, collection): current_fn = self._get_file(collection, permanent=False) @@ -108,8 +110,8 @@ def _make_id(self): return str(uuid4()) @classmethod - def permanently_erase_database(cls, db_name, storage_dir='json_store'): + def permanently_erase_database(cls, db_name, storage_dir=None): # Clear out any .json files. - storage_dir = os.path.join(os.environ['PANDIR'], storage_dir, db_name) + storage_dir = os.path.join(storage_dir, db_name) for f in glob(os.path.join(storage_dir, '*.json')): os.remove(f) diff --git a/src/panoptes/utils/utils.py b/src/panoptes/utils/utils.py index 6cea7666b..c105406aa 100644 --- a/src/panoptes/utils/utils.py +++ b/src/panoptes/utils/utils.py @@ -77,14 +77,13 @@ def get_free_space(directory=None): Args: - directory (str, optional): Path to directory. If None defaults to $PANDIR. + directory (str, optional): Path to directory. If None defaults to root. Returns: astropy.units.Quantity: The number of gigabytes avialable in folder. """ - if directory is None: - directory = os.getenv('PANDIR') + directory = directory or os.path.abspath('/') _, _, free_space = shutil.disk_usage(directory) free_space = (free_space * u.byte).to(u.gigabyte) diff --git a/tests/Dockerfile b/tests/Dockerfile index 35d401199..1cfa75a01 100644 --- a/tests/Dockerfile +++ b/tests/Dockerfile @@ -4,9 +4,6 @@ ENV DEBIAN_FRONTEND=noninteractive ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 ENV SHELL /bin/bash -ARG pan_dir=/var/panoptes -ENV PANDIR $pan_dir - ARG pip_extras="[config,images,testing,social]" WORKDIR /tmp @@ -19,10 +16,10 @@ RUN apt-get update && \ dcraw exiftool libcfitsio-dev libcfitsio-bin \ libfreetype6-dev libpng-dev libjpeg-dev libffi-dev -COPY environment.yaml . +COPY docker/environment.yaml . RUN conda env update -n base -f environment.yaml -WORKDIR "${PANDIR}/panoptes-utils" +WORKDIR /app COPY . . RUN pip install -e ".${pip_extras}" && \ # Cleanup diff --git a/tests/images/test_image_utils.py b/tests/images/test_image_utils.py index c0f2e5628..51cbb58d8 100644 --- a/tests/images/test_image_utils.py +++ b/tests/images/test_image_utils.py @@ -1,11 +1,9 @@ import os -import numpy as np -import pytest -import shutil import tempfile +import numpy as np +import pytest from astropy.nddata import Cutout2D - from panoptes.utils import images as img_utils from panoptes.utils import error @@ -49,9 +47,8 @@ def test_make_pretty_image(solved_fits_file, tiny_fits_file, save_environ): imgdir = os.path.join(tmpdir, 'images') assert not os.path.isdir(imgdir) os.makedirs(imgdir, exist_ok=True) - os.environ['PANDIR'] = tmpdir - link_path = os.path.expandvars('$PANDIR/latest.jpg') + link_path = os.path.join(tmpdir, 'latest.jpg') pretty = img_utils.make_pretty_image(solved_fits_file, link_path=link_path) assert pretty assert os.path.isfile(pretty) @@ -82,8 +79,8 @@ def test_make_pretty_image_cr2_fail(): img_utils.make_pretty_image(tmpfile) -def test_make_pretty_image_cr2(cr2_file): - link_path = os.path.expandvars('$PANDIR/images/latest.jpg') +def test_make_pretty_image_cr2(cr2_file, tmpdir): + link_path = str(tmpdir.mkdir('images').join('latest.jpg')) pretty_path = img_utils.make_pretty_image(cr2_file, title='CR2 Test', image_type='cr2', diff --git a/tests/test_database.py b/tests/test_database.py index 7a08d5b6c..a78e8a443 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -67,13 +67,26 @@ def test_warn_bad_object(db): db.insert('observations', {'junk': db}) -def test_delete_file_db(): +def test_delete_file_db(tmpdir): with pytest.raises(Exception): - PanDB.permanently_erase_database('memory', 'panoptes_testing', really='Nope', dangerous='Hopefully not') + PanDB.permanently_erase_database('memory', + 'panoptes_testing', + really='Nope', + dangerous='Hopefully not') with pytest.raises(ValueError): - PanDB.permanently_erase_database('memory', 'do_not_delete_me', really='Nope', dangerous='Again, we hope not') - - PanDB.permanently_erase_database('file', 'panoptes_testing', storage_dir='testing', really='Yes', + PanDB.permanently_erase_database('memory', + 'do_not_delete_me', + really='Nope', + dangerous='Again, we hope not') + + file_dir = tmpdir.mkdir('testing') + PanDB.permanently_erase_database('file', + 'panoptes_testing', + storage_dir=str(file_dir), + really='Yes', dangerous='Totally') - PanDB.permanently_erase_database('memory', 'panoptes_testing', dangerous='Totally', really='Yes') + PanDB.permanently_erase_database('memory', + 'panoptes_testing', + dangerous='Totally', + really='Yes') diff --git a/tests/testing.yaml b/tests/testing.yaml index 45ba13dbf..997490c16 100644 --- a/tests/testing.yaml +++ b/tests/testing.yaml @@ -25,7 +25,7 @@ location: timezone: US/Hawaii gmt_offset: -600 # Offset in minutes from GMT during. directories: - base: /var/panoptes + base: /app images: images data: data resources: POCS/resources/ @@ -66,7 +66,7 @@ cameras: # # An example folder structure would be: # -# $PANDIR/images/fields/Hd189733/14d3bd/20180901T120001/ +# /images/fields/Hd189733/14d3bd/20180901T120001/ # # In this folder will be stored JPG and FITS images. A timelapse of the # observation can be made (one per camera) and the JPGs optionally removed