From 7f63092e5538ff317db19059f163831ccc7b0b59 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 25 Jul 2023 15:52:52 +0200 Subject: [PATCH 01/10] Add multiple build platforms --- .github/workflows/release_docker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_docker.yaml b/.github/workflows/release_docker.yaml index 6ceb1d060..3d5296ec8 100644 --- a/.github/workflows/release_docker.yaml +++ b/.github/workflows/release_docker.yaml @@ -43,6 +43,7 @@ jobs: push: true tags: ${{ steps.meta_dockerhub.outputs.tags }} labels: ${{ steps.meta_dockerhub.outputs.labels }} + platforms: linux/amd64,linux/arm64 - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} From 73a7940c7ba9bb9ebeec40e82b3b655c50ccca67 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 27 Jul 2023 16:01:28 +0200 Subject: [PATCH 02/10] Automatically update Dockerhub description --- .github/workflows/release_docker.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/release_docker.yaml b/.github/workflows/release_docker.yaml index 3d5296ec8..85c70987a 100644 --- a/.github/workflows/release_docker.yaml +++ b/.github/workflows/release_docker.yaml @@ -44,6 +44,15 @@ jobs: tags: ${{ steps.meta_dockerhub.outputs.tags }} labels: ${{ steps.meta_dockerhub.outputs.labels }} platforms: linux/amd64,linux/arm64 + + - name: Update repo description + uses: peter-evans/dockerhub-description@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: openml/openml-python + short-description: "pre-installed openml-python environment" + readme-filepath: ./docker/readme.md - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} From 998e641d7db9008f3ef29b6c74997c959344dbb7 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 27 Jul 2023 18:19:13 +0300 Subject: [PATCH 03/10] Launch Python instead of Bash by default --- docker/startup.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docker/startup.sh b/docker/startup.sh index 2a75a621c..45bcf7fc9 100644 --- a/docker/startup.sh +++ b/docker/startup.sh @@ -1,3 +1,6 @@ +# Entry script to switch between the different Docker functionalities. +# By default, execute Python with OpenML pre-installed +# # Entry script to allow docker to be ran for bash, tests and docs. # The script assumes a code repository can be mounted to ``/code`` and an output directory to ``/output``. # Executes ``mode`` on ``branch`` or the provided ``code`` directory. @@ -10,10 +13,10 @@ # Can be a branch on a Github fork, specified with the USERNAME#BRANCH format. # The test or doc build is executed on this branch. -if [ -z "$1" ]; then - echo "Executing in BASH mode." - bash - exit +if [[ ! ( $1 = "doc" || $1 = "test" ) ]]; then + source /omlp/venv/bin/activate + python "$@" + exit 0 fi # doc and test modes require mounted directories and/or specified branches @@ -73,4 +76,4 @@ if [ "$1" == "doc" ]; then if [ -d "/output" ]; then cp -r /omlp/doc/build /output fi -fi +fi \ No newline at end of file From d57cd5d8adc6cd7771bf5be625f0fc7d3e3f53bb Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 27 Jul 2023 18:22:42 +0300 Subject: [PATCH 04/10] Change `omlp` directory name to less cryptic `openml` --- docker/Dockerfile | 4 ++-- docker/startup.sh | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c27abba40..3f72f3598 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,8 +2,8 @@ # Useful building docs or running unix tests from a Windows host. FROM python:3.10 -RUN git clone https://github.com/openml/openml-python.git omlp -WORKDIR omlp +RUN git clone https://github.com/openml/openml-python.git openml +WORKDIR openml RUN python -m venv venv RUN venv/bin/pip install wheel setuptools RUN venv/bin/pip install -e .[test,examples,docs,examples_unix] diff --git a/docker/startup.sh b/docker/startup.sh index 45bcf7fc9..2f006549e 100644 --- a/docker/startup.sh +++ b/docker/startup.sh @@ -14,7 +14,7 @@ # The test or doc build is executed on this branch. if [[ ! ( $1 = "doc" || $1 = "test" ) ]]; then - source /omlp/venv/bin/activate + source /openml/venv/bin/activate python "$@" exit 0 fi @@ -35,8 +35,8 @@ if [ "$1" == "doc" ] && [ -n "$2" ] && ! [ -d "/output" ]; then fi if [ -n "$2" ]; then - # if a branch is provided, we will pull it into the `omlp` local repository that was created with the image. - cd omlp + # if a branch is provided, we will pull it into the `openml` local repository that was created with the image. + cd openml if [[ $2 == *#* ]]; then # If a branch is specified on a fork (with NAME#BRANCH format), we have to construct the url before pulling # We add a trailing '#' delimiter so the second element doesn't get the trailing newline from <<< @@ -55,12 +55,12 @@ if [ -n "$2" ]; then exit 1 fi git pull - code_dir="/omlp" + code_dir="/openml" else code_dir="/code" fi -source /omlp/venv/bin/activate +source /openml/venv/bin/activate cd $code_dir # The most recent ``main`` is already installed, but we want to update any outdated dependencies pip install -e .[test,examples,docs,examples_unix] @@ -74,6 +74,6 @@ if [ "$1" == "doc" ]; then make html make linkcheck if [ -d "/output" ]; then - cp -r /omlp/doc/build /output + cp -r /openml/doc/build /output fi fi \ No newline at end of file From 7709348a22db1e67b9c2268ecab8e63fc133f629 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 27 Jul 2023 20:26:23 +0300 Subject: [PATCH 05/10] Change directory to `openml` for running purpose of running script For mounted scripts, instructions say to mount them to `/openml`, so we have to `cd` before invoking `python`. --- docker/startup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/startup.sh b/docker/startup.sh index 2f006549e..34a5c61f3 100644 --- a/docker/startup.sh +++ b/docker/startup.sh @@ -14,7 +14,8 @@ # The test or doc build is executed on this branch. if [[ ! ( $1 = "doc" || $1 = "test" ) ]]; then - source /openml/venv/bin/activate + cd openml + source venv/bin/activate python "$@" exit 0 fi From 7412f056dd0952c9eb740554d9e8e824fdf811fc Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 27 Jul 2023 20:27:44 +0300 Subject: [PATCH 06/10] Update readme to reflect updates (python by default, rename dirs) --- docker/readme.md | 149 ++++++++++++++++++++++++++++++----------------- 1 file changed, 94 insertions(+), 55 deletions(-) diff --git a/docker/readme.md b/docker/readme.md index 47ad6d23b..01e4e15e9 100644 --- a/docker/readme.md +++ b/docker/readme.md @@ -1,86 +1,125 @@ # OpenML Python Container -This docker container has the latest development version of openml-python downloaded and pre-installed. -It can be used to run the unit tests or build the docs in a fresh and/or isolated unix environment. -Instructions only tested on a Windows host machine. +This docker container has the latest version of openml-python downloaded and pre-installed. +It can also be used by developers to run unit tests or build the docs in +a fresh and/or isolated unix environment. +This document contains information about: -First pull the docker image: - - docker pull openml/openml-python + 1. [Usage](#usage) take you over + 2. [Using local or remote code](#using-local-or-remote-code) is useful when testing your own latest changes + 3. [Versions](#versions) help you identify which image to use. + 4. [Developer](#for-developers) information about the Docker image. ## Usage +There are three main ways to use the image: running a pre-installed Python environment, +running tests, and building documentation. - docker run -it openml/openml-python [DOC,TEST] [BRANCH] +### Running `Python` with pre-installed `OpenML-Python` (default): -The image is designed to work with two specified directories which may be mounted ([`docker --mount documentation`](https://docs.docker.com/storage/bind-mounts/#start-a-container-with-a-bind-mount)). -You can mount your openml-python folder to the `/code` directory to run tests or build docs on your local files. -You can mount an `/output` directory to which the container will write output (currently only used for docs). -Each can be mounted by adding a `--mount type=bind,source=SOURCE,destination=/DESTINATION` where `SOURCE` is the absolute path to your code or output directory, and `DESTINATION` is either `code` or `output`. - -E.g. mounting a code directory: +To run `Python` with a pre-installed `OpenML-Python` environment run: - docker run -i --mount type=bind,source="E:\\repositories/openml-python",destination="/code" -t openml/openml-python +```text +docker run -it openml/openml-python +``` -E.g. mounting an output directory: +this accepts the normal `Python` arguments, e.g.: - docker run -i --mount type=bind,source="E:\\files/output",destination="/output" -t openml/openml-python +```text +docker run openml/openml-python -c "import openml; print(openml.__version__)" +``` -You can mount both at the same time. +if you want to run a local script, it needs to be mounted first. Mount it into the +`openml` folder: -### Bash (default) -By default bash is invoked, you should also use the `-i` flag when starting the container so it processes input: +``` +docker run -v PATH/TO/FILE:/openml/MY_SCRIPT.py openml/openml-python MY_SCRIPT.py +``` - docker run -it openml/openml-python +### Running unit tests -### Building Documentation -There are two ways to build documentation, either directly from the `HEAD` of a branch on Github or from your local directory. +You can run the unit tests by passing `test` as the first argument: -#### Building from a local repository -Building from a local directory requires you to mount it to the ``/code`` directory: +```text +docker run openml/openml-python test +``` - docker run --mount type=bind,source=PATH_TO_REPOSITORY,destination=/code -t openml/openml-python doc +see [the section below]((#using-local-or-remote-code) for running against local changes +or a remote branch. -The produced documentation will be in your repository's ``doc/build`` folder. -If an `/output` folder is mounted, the documentation will *also* be copied there. +### Building documentation -#### Building from an online repository -Building from a remote repository requires you to specify a branch. -The branch may be specified by name directly if it exists on the original repository (https://github.com/openml/openml-python/): +You can build the documentation by passing `doc` as the first argument, +you should [mount]((https://docs.docker.com/storage/bind-mounts/#start-a-container-with-a-bind-mount)) +an output directory in which the docs will be stored, on Windows: - docker run --mount type=bind,source=PATH_TO_OUTPUT,destination=/output -t openml/openml-python doc BRANCH +```text + docker run --mount type=bind,source="E:\\files/output",destination="/output" openml/openml-python doc +``` -Where `BRANCH` is the name of the branch for which to generate the documentation. -It is also possible to build the documentation from the branch on a fork, in this case the `BRANCH` should be specified as `GITHUB_NAME#BRANCH` (e.g. `PGijsbers#my_feature`) and the name of the forked repository should be `openml-python`. +on Linux: +```text + docker run --mount type=bind,source="./output",destination="/output" openml/openml-python doc +``` + +see [the section below]((#using-local-or-remote-code) for running against local changes +or a remote branch. -### Running tests -There are two ways to run tests, either directly from the `HEAD` of a branch on Github or from your local directory. -It works similar to building docs, but should specify `test` as mode. -For example, to run tests on your local repository: +*Note: you can forgo mounting an output directory to test if the docs build successfully, +but the result will only be available within the docker container under `/openml/docs/build`.* - docker run --mount type=bind,source=PATH_TO_REPOSITORY,destination=/code -t openml/openml-python test - -Running tests from the state of an online repository is supported similar to building documentation (i.e. specify `BRANCH` instead of mounting `/code`). - -## Troubleshooting +## Using local or remote code + +You can build docs or run tests against your local repository or a Github repository. +In the examples below, change the `source` to match the location of your local repository. + +### Using a local repository + +To use a local directory, mount it in the `/code` directory, on Windows: -When you are mounting a directory you can check that it is mounted correctly by running the image in bash mode. -Navigate to the `/code` and `/output` directories and see if the expected files are there. -If e.g. there is no code in your mounted `/code`, you should double-check the provided path to your host directory. +```text + docker run --mount type=bind,source="E:\\repositories/openml-python",destination="/code" openml/openml-python test +``` -## Notes for developers -This section contains some notes about the structure of the image, intended for those who want to work on it. +on Linux: +```text + docker run --mount type=bind,source="~/repositories/openml-python",destination="/code" openml/openml-python test +``` + +when building docs, you also need to mount an output directory as shown above, so add both: + +```text +docker run --mount type=bind,source="./output",destination="/output" --mount type=bind,source="~/repositories/openml-python",destination="/code" openml/openml-python doc +``` + +### Using a Github repository +Building from a remote repository requires you to specify a branch. +The branch may be specified by name directly if it exists on the original repository (https://github.com/openml/openml-python/): + + docker run --mount type=bind,source=PATH_TO_OUTPUT,destination=/output openml/openml-python [test,doc] BRANCH + +Where `BRANCH` is the name of the branch for which to generate the documentation. +It is also possible to build the documentation from the branch on a fork, +in this case the `BRANCH` should be specified as `GITHUB_NAME#BRANCH` (e.g. +`PGijsbers#my_feature_branch`) and the name of the forked repository should be `openml-python`. + +## For developers +This section contains some notes about the structure of the image, +intended for those who want to work on it. ### Added Directories The `openml/openml-python` image is built on a vanilla `python:3` image. -Additionally it contains the following files are directories: - - - `/omlp`: contains the openml-python repository in the state with which the image was built by default. - If working with a `BRANCH`, this repository will be set to the `HEAD` of `BRANCH`. - - `/omlp/venv/`: contains the used virtual environment for `doc` and `test`. It has `openml-python` dependencies pre-installed. - When invoked with `doc` or `test`, the dependencies will be updated based on the `setup.py` of the `BRANCH` or mounted `/code`. +Additionally, it contains the following files are directories: + + - `/openml`: contains the openml-python repository in the state with which the image + was built by default. If working with a `BRANCH`, this repository will be set to + the `HEAD` of `BRANCH`. + - `/openml/venv/`: contains the used virtual environment for `doc` and `test`. It has + `openml-python` dependencies pre-installed. When invoked with `doc` or `test`, the + dependencies will be updated based on the `setup.py` of the `BRANCH` or mounted `/code`. - `/scripts/startup.sh`: the entrypoint of the image. Takes care of the automated features (e.g. `doc` and `test`). ## Building the image -To build the image yourself, execute `docker build -f Dockerfile .` from this directory. -It will use the `startup.sh` as is, so any local changes will be present in the image. +To build the image yourself, execute `docker build -f Dockerfile .` from the `docker` +directory of the `openml-python` repository. It will use the `startup.sh` as is, so any +local changes will be present in the image. From baa754fe68efb98d553a46505d109df5902c1b1a Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 27 Jul 2023 20:44:33 +0300 Subject: [PATCH 07/10] Add branch/code for doc and test examples as they are required --- docker/readme.md | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/docker/readme.md b/docker/readme.md index 01e4e15e9..e50b78084 100644 --- a/docker/readme.md +++ b/docker/readme.md @@ -38,28 +38,31 @@ docker run -v PATH/TO/FILE:/openml/MY_SCRIPT.py openml/openml-python MY_SCRIPT.p ### Running unit tests -You can run the unit tests by passing `test` as the first argument: +You can run the unit tests by passing `test` as the first argument. +It also requires a local or remote repository to be specified, which is explained +[below]((#using-local-or-remote-code). For this example, we specify to test the +`develop` branch: ```text -docker run openml/openml-python test +docker run openml/openml-python test develop ``` -see [the section below]((#using-local-or-remote-code) for running against local changes -or a remote branch. - ### Building documentation You can build the documentation by passing `doc` as the first argument, you should [mount]((https://docs.docker.com/storage/bind-mounts/#start-a-container-with-a-bind-mount)) -an output directory in which the docs will be stored, on Windows: +an output directory in which the docs will be stored. You also need to provide a remote +or local repository as explained in [the section below]((#using-local-or-remote-code). +In this example, we build documentation for the `develop` branch. +On Windows: ```text - docker run --mount type=bind,source="E:\\files/output",destination="/output" openml/openml-python doc + docker run --mount type=bind,source="E:\\files/output",destination="/output" openml/openml-python doc develop ``` on Linux: ```text - docker run --mount type=bind,source="./output",destination="/output" openml/openml-python doc + docker run --mount type=bind,source="./output",destination="/output" openml/openml-python doc develop ``` see [the section below]((#using-local-or-remote-code) for running against local changes @@ -83,13 +86,13 @@ To use a local directory, mount it in the `/code` directory, on Windows: on Linux: ```text - docker run --mount type=bind,source="~/repositories/openml-python",destination="/code" openml/openml-python test + docker run --mount type=bind,source="/Users/pietergijsbers/repositories/openml-python",destination="/code" openml/openml-python test ``` when building docs, you also need to mount an output directory as shown above, so add both: ```text -docker run --mount type=bind,source="./output",destination="/output" --mount type=bind,source="~/repositories/openml-python",destination="/code" openml/openml-python doc +docker run --mount type=bind,source="./output",destination="/output" --mount type=bind,source="/Users/pietergijsbers/repositories/openml-python",destination="/code" openml/openml-python doc ``` ### Using a Github repository From 21161b6c121fb90865de6f8112d6b6d45747b4c4 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 27 Jul 2023 20:53:54 +0300 Subject: [PATCH 08/10] Ship docker images with readme --- docker/Dockerfile | 2 ++ docker/readme.md | 3 +++ 2 files changed, 5 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3f72f3598..a84723309 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,6 +11,8 @@ RUN venv/bin/pip install -e .[test,examples,docs,examples_unix] WORKDIR / RUN mkdir scripts ADD startup.sh scripts/ +ADD readme.md / + # Due to the nature of the Docker container it might often be built from Windows. # It is typical to have the files with \r\n line-ending, we want to remove it for the unix image. RUN sed -i 's/\r//g' scripts/startup.sh diff --git a/docker/readme.md b/docker/readme.md index e50b78084..bd1744ea2 100644 --- a/docker/readme.md +++ b/docker/readme.md @@ -10,6 +10,9 @@ This document contains information about: 3. [Versions](#versions) help you identify which image to use. 4. [Developer](#for-developers) information about the Docker image. +*note:* each docker image is shipped with a readme, which you can read with: +`docker run --entrypoint=/bin/cat openml/openml-python:TAG readme.md` + ## Usage There are three main ways to use the image: running a pre-installed Python environment, From e279c2b2e419a6cd14f5659015f6dfb6a9e3cd09 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 27 Jul 2023 21:07:56 +0300 Subject: [PATCH 09/10] Only update readme on release, also try build docker on PR --- .github/workflows/release_docker.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_docker.yaml b/.github/workflows/release_docker.yaml index 85c70987a..1b139c978 100644 --- a/.github/workflows/release_docker.yaml +++ b/.github/workflows/release_docker.yaml @@ -3,9 +3,13 @@ name: release-docker on: push: branches: - - 'main' - 'develop' - 'docker' + tags: + - 'v*' + pull_request: + branches: + - 'develop' jobs: @@ -21,6 +25,7 @@ jobs: uses: docker/setup-buildx-action@v2 - name: Login to DockerHub + if: github.event_name != 'pull_request' uses: docker/login-action@v2 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -40,12 +45,13 @@ jobs: uses: docker/build-push-action@v4 with: context: ./docker/ - push: true tags: ${{ steps.meta_dockerhub.outputs.tags }} labels: ${{ steps.meta_dockerhub.outputs.labels }} platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name == 'push' }} - name: Update repo description + if: ${{ startsWith(github.ref, 'refs/tags/v') }} uses: peter-evans/dockerhub-description@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} From 1c9bda74125d05d562c6fe4d057f739cde2f903d Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 15 Aug 2023 17:54:02 +0300 Subject: [PATCH 10/10] Update the toc descriptions --- docker/readme.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/readme.md b/docker/readme.md index bd1744ea2..d0af9d9fe 100644 --- a/docker/readme.md +++ b/docker/readme.md @@ -5,10 +5,10 @@ It can also be used by developers to run unit tests or build the docs in a fresh and/or isolated unix environment. This document contains information about: - 1. [Usage](#usage) take you over - 2. [Using local or remote code](#using-local-or-remote-code) is useful when testing your own latest changes - 3. [Versions](#versions) help you identify which image to use. - 4. [Developer](#for-developers) information about the Docker image. + 1. [Usage](#usage): how to use the image and its main modes. + 2. [Using local or remote code](#using-local-or-remote-code): useful when testing your own latest changes. + 3. [Versions](#versions): identify which image to use. + 4. [Development](#for-developers): information about the Docker image for developers. *note:* each docker image is shipped with a readme, which you can read with: `docker run --entrypoint=/bin/cat openml/openml-python:TAG readme.md`