Merge branch 'feature/docker-testing' into 'master'

add testing environment specification as Dockerfiles and conda environments

See merge request kwant/kwant!260

.gitlab-ci.yml

-image: gitlab.kwant-project.org:5005/kwant/testing
+image: gitlab.kwant-project.org:5005/kwant/kwant
 
 stages:
+  - build-env
   - build
   - test
   - deploy
@@ -11,6 +12,69 @@ variables:
   # secret information, and using 'ssh-keyscan' causes the CI server's IP to be blacklisted
   IGNORE_HOSTKEY: "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
 
+## Building Docker environments
+## Only runs when docker specifications change
+
+.build-env: &build-env
+  stage: build-env
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
+  artifacts:
+    untracked: true
+    expire_in: 1 hour
+  before_script:
+    - mkdir -p /root/.docker
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /root/.docker/config.json
+
+build-env:ubuntu:
+  <<: *build-env
+  only:
+    changes:
+      - docker/Dockerfile.ubuntu
+  script:
+    - /kaniko/executor
+      --context $CI_PROJECT_DIR/docker
+      --dockerfile $CI_PROJECT_DIR/docker/Dockerfile.ubuntu
+      --destination $CI_REGISTRY_IMAGE/ubuntu
+
+build-env:debian:
+  <<: *build-env
+  only:
+    changes:
+      - docker/Dockerfile.debian
+  script:
+    - /kaniko/executor
+      --context $CI_PROJECT_DIR/docker
+      --dockerfile $CI_PROJECT_DIR/docker/Dockerfile.debian
+      --destination $CI_REGISTRY_IMAGE/debian
+
+build-env:conda:
+  <<: *build-env
+  only:
+    changes:
+      - docker/Dockerfile.conda
+      - docker/*.yml
+  script:
+    - /kaniko/executor
+      --context $CI_PROJECT_DIR/docker
+      --dockerfile $CI_PROJECT_DIR/docker/Dockerfile.conda
+      --destination $CI_REGISTRY_IMAGE/conda
+
+build-env:default:
+  <<: *build-env
+  only:
+    changes:
+      - docker/Dockerfile.conda
+      - docker/*.yml
+  script:
+    - /kaniko/executor
+      --context $CI_PROJECT_DIR/docker
+      --dockerfile $CI_PROJECT_DIR/docker/Dockerfile.conda
+      --destination $CI_REGISTRY_IMAGE
+
+
+## Build environment specifications
 
 .build: &build
   stage: build
@@ -20,7 +84,7 @@ variables:
     - ./setup.py build_ext -i
   artifacts:
     untracked: true
-    expire_in: 1 hour
+    expire_in: 2 hour
 
 
 .stable-env: &stable_env
@@ -49,10 +113,10 @@ variables:
     - source activate kwant-latest
 
 .ubuntu-env: &ubuntu_env
-  image: gitlab.kwant-project.org:5005/kwant/testing/ubuntu
+  image: gitlab.kwant-project.org:5005/kwant/kwant/ubuntu
 
 .debian-env: &debian_env
-  image: gitlab.kwant-project.org:5005/kwant/testing/debian
+  image: gitlab.kwant-project.org:5005/kwant/kwant/debian
 
 
 ## Build Jobs

docker/Dockerfile.conda

+FROM conda/miniconda3:latest
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        # all the hard non-Python dependencies
+        git make patch build-essential \
+        # Additional tools for running CI
+        file rsync openssh-client \
+        # Documentation building
+        inkscape texlive-full zip \
+    && apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY kwant-latest.yml kwant-stable.yml kwant-stable-no-extras.yml /
+
+RUN conda env create -qf kwant-stable.yml
+RUN conda env create -qf kwant-stable-no-extras.yml
+RUN conda env create -qf kwant-latest.yml

docker/Dockerfile.debian

+FROM debian:latest
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        gnupg dirmngr apt-transport-https ca-certificates curl software-properties-common
+
+RUN echo "deb http://downloads.kwant-project.org/debian/ stretch-backports main" >> /etc/apt/sources.list && \
+    apt-key adv --no-tty --keyserver pool.sks-keyservers.net --recv-key C3F147F5980F3535 && \
+    apt-get update && apt-get install -y --no-install-recommends \
+        # all the hard non-Python dependencies
+        git g++ make patch gfortran libblas-dev liblapack-dev \
+        libmumps-scotch-dev pkg-config libfreetype6-dev \
+        # all the hard Python dependencies
+        python3-all-dev python3-setuptools python3-pip python3-tk python3-wheel \
+        python3-numpy python3-scipy python3-matplotlib python3-sympy python3-tinyarray \
+        # Additional tools for running CI
+        file rsync openssh-client \
+    && apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+### install build and testing dependencies
+RUN pip3 install \
+      cython \
+      pytest \
+      pytest-runner \
+      pytest-cov \
+      pytest-flakes \
+      pytest-pep8

docker/Dockerfile.ubuntu

+FROM ubuntu:16.04
+
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        gnupg dirmngr apt-transport-https software-properties-common
+RUN apt-add-repository -s ppa:kwant-project/ppa && \
+    apt-get update && apt-get install -y --no-install-recommends \
+        # all the hard non-Python dependencies
+        git g++ make patch gfortran libblas-dev liblapack-dev \
+        libmumps-scotch-dev pkg-config libfreetype6-dev \
+        # all the hard Python dependencies
+        python3-all-dev python3-setuptools python3-pip python3-tk python3-wheel \
+        python3-numpy python3-scipy python3-matplotlib python3-sympy python3-tinyarray \
+        # Additional tools for running CI
+        file rsync openssh-client \
+    && apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+### install build and testing dependencies
+RUN pip3 install \
+      cython \
+      pytest \
+      pytest-runner \
+      pytest-cov \
+      pytest-flakes \
+      pytest-pep8

docker/README.rst

+====================
+Kwant Testing Images
+====================
+This directory contains files for building Docker images used in
+testing Kwant. The Dockerfile for a given platforms can also
+be understood as recipes for how to install the prerequesites of
+Kwant on that platform. To build an image it is enough to execute::
+
+    docker build -f <dockerfile>
+
+from this directory (where ``<dockerfile>`` should be replaced with the
+filename of the appropriate dockerfile to build).
+
+Dockerfiles
+===========
+
+Dockerfile.debian
+  Builds an environment based on Debian stable. Python 3 and all the dependencies
+  of Kwant are installed using the Debian repositories. Includes all optional
+  dependencies (for plotting and the continuum module).
+
+Dockerfile.ubuntu
+  Builds an environment based on the latest Ubuntu LTS release that is supported
+  by Kwant (we typically wait until 6 months after a new LTS release to upgrade
+  Kwant's requirements, to give people time to switch). Python 3 and
+  all the dependencies of Kwant are installed using the Ubuntu repositories.
+  Includes all optional dependencies (for plotting and the continuum module).
+
+Dockerfile.conda
+  Builds an environment that contains Miniconda and a minimal number of
+  system-installed packages (mainly to make Git and documentation building work).
+  Individual testing environments are installed as conda environments (see
+  the next section for details).
+
+Conda Environments
+==================
+These conda environments are specified in Yaml files, and are loaded into
+the conda-based Docker image. All the dependencies are loaded from the
+`conda-forge <https://anaconda.org/conda-forge/>`_ Anaconda channel.
+
+kwant-stable-no-extras
+  The minimal environment in which Kwant can run. Pins all the dependencies to
+  the oldest supported versions, and does not include optional dependencies.
+
+kwant-stable
+  Pins all the dependencies to the oldest supported versions, including all
+  optional dependencies.
+
+kwant-latest
+  References all Kwant dependencies, but does not pin any versions, meaning
+  that the latest released versions of the dependencies are used (at least,
+  the latest released versions on conda-forge)

docker/kwant-latest.yml

+name: kwant-latest
+channels:
+    - conda-forge
+dependencies:
+    - python=3.6
+    - numpy
+    - scipy
+    - tinyarray
+    - sympy
+    - matplotlib
+    # Linear algebra libraraies
+    - mumps
+    - blas #=1.1 openblas
+    - openblas #=0.2.20
+    # Build toolchain
+    - toolchain
+    - cython
+    - gcc
+    - libgfortran
+    ## Dev Dependencies
+    # Testing
+    - pytest
+    - pytest-cov
+    - pytest-flakes
+    - pytest-pep8
+    # Documentation building
+    - sphinx=1.7.4  # later versions seem to have problems
+    - numpydoc
+    - requests

docker/kwant-stable-no-extras.yml

+name: kwant-stable-no-extras
+channels:
+    - conda-forge
+dependencies:
+    - python=3.5
+    - numpy=1.11.0
+    - scipy=0.17  # No minor version because conda-forge has no 0.17.0
+    - tinyarray=1.2.0
+    # Linear algebra libraraies
+    - blas #=1.1 openblas
+    - openblas #=0.2.20
+    # Build toolchain
+    - toolchain
+    - cython
+    - gcc
+    - libgfortran
+    ## Dev Dependencies
+    # Testing
+    - pytest
+    - pytest-cov
+    - pytest-flakes
+    - pytest-pep8
+    # Documentation building
+    - sphinx=1.7.4  # later versions seem to have problems
+    - numpydoc
+    - requests

docker/kwant-stable.yml

+name: kwant-stable
+channels:
+    - conda-forge
+dependencies:
+    - python=3.5
+    - numpy=1.11.0
+    - scipy=0.17  # No minor version because conda-forge has no 0.17.0
+    - tinyarray=1.2.0
+    - sympy=0.7.6
+    - matplotlib=1.5.1
+    # Linear algebra libraraies
+    - mumps
+    - blas #=1.1 openblas
+    - openblas #=0.2.20
+    # Build toolchain
+    - toolchain
+    - cython
+    - gcc
+    - libgfortran
+    ## Dev Dependencies
+    # Testing
+    - pytest
+    - pytest-cov
+    - pytest-flakes
+    - pytest-pep8
+    # Documentation building
+    - sphinx=1.7.4  # later versions seem to have problems
+    - numpydoc
+    - requests