python_cron.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Python Cron

on:
  push:
    paths:
      - '.github/workflows/python_cron.yml'
  pull_request:
    paths:
      - '.github/workflows/python_cron.yml'
  schedule:
    - cron: |
        0 */12 * * *

env:
  DOCKER_BUILDKIT: 0
  COMPOSE_DOCKER_CLI_BUILD: 1
  ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }}
  ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }}

jobs:

  docker:
    name: ${{ matrix.title }}
    runs-on: ubuntu-latest
    if: ${{ !contains(github.event.pull_request.title, 'WIP') && github.repository == 'apache/arrow' }}
    strategy:
      fail-fast: false
      matrix:
        name:
          - debian-10-python-3
          - fedora-33-python-3
          - ubuntu-18.04-python-3
          - conda-python-3.7-dask-latest
          - conda-python-3.7-turbodbc-latest
          - conda-python-3.7-kartothek-latest
          - conda-python-3.7-pandas-0.24
          - conda-python-3.7-pandas-master
          - conda-python-3.7-hdfs-2.9.2
        include:
          - name: debian-10-python-3
            cache: debian-10-python-3
            image: debian-python
            title: AMD64 Debian 10 Python 3
            debian: 10
          - name: fedora-33-python-3
            cache: fedora-33-python-3
            image: fedora-python
            title: AMD64 Fedora 33 Python 3
            fedora: 33
          - name: ubuntu-18.04-python-3
            cache: ubuntu-18.04-python-3
            image: ubuntu-python
            title: AMD64 Ubuntu 18.04 Python 3
            ubuntu: 18.04
          - name: conda-python-3.7-dask-latest
            cache: conda-python-3.7
            image: conda-python-dask
            title: AMD64 Conda Python 3.7 Dask latest
            dask: latest
          - name: conda-python-3.7-turbodbc-latest
            cache: conda-python-3.7
            image: conda-python-turbodbc
            title: AMD64 Conda Python 3.7 Turbodbc latest
            turbodbc: latest
          - name: conda-python-3.7-kartothek-latest
            cache: conda-python-3.7
            image: conda-python-kartothek
            title: AMD64 Conda Python 3.7 Kartothek latest
            kartothek: latest
          - name: conda-python-3.7-pandas-0.24
            cache: conda-python-3.7
            image: conda-python-pandas
            title: AMD64 Conda Python 3.7 Pandas 0.24
            pandas: 0.24
          - name: conda-python-3.7-pandas-master
            cache: conda-python-3.7
            image: --no-leaf-cache conda-python-pandas
            title: AMD64 Conda Python 3.7 Pandas master
            pandas: master
          - name: conda-python-3.7-hdfs-2.9.2
            cache: conda-python-3.7
            image: conda-python-hdfs
            title: AMD64 Conda Python 3.7 HDFS 2.9.2
            hdfs: 2.9.2
    env:
      # the defaults here should correspond to the values in .env
      DEBIAN: ${{ matrix.debian || 10 }}
      FEDORA: ${{ matrix.fedora || 33 }}
      UBUNTU: ${{ matrix.ubuntu || 18.04 }}
      PYTHON: ${{ matrix.python || 3.7 }}
      HDFS: ${{ matrix.hdfs || '2.9.2' }}
      DASK: ${{ matrix.dask || 'latest' }}
      TURBODBC: ${{ matrix.turbodbc || 'latest' }}
      PANDAS: ${{ matrix.pandas || 'latest' }}
      KARTOTHEK: ${{ matrix.kartothek || 'latest' }}
    steps:
      - name: Checkout Arrow
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
      - name: Fetch Submodules and Tags
        run: ci/scripts/util_checkout.sh
      - name: Free Up Disk Space
        run: ci/scripts/util_cleanup.sh
      - name: Cache Docker Volumes
        uses: actions/cache@v1
        with:
          path: .docker
          key: ${{ matrix.cache }}-${{ hashFiles('cpp/**') }}
          restore-keys: ${{ matrix.cache }}-
      - name: Setup Python
        uses: actions/setup-python@v1
        with:
          python-version: 3.8
      - name: Setup Archery
        run: pip install -e dev/archery[docker]
      - name: Execute Docker Build
        run: |
          sudo sysctl -w kernel.core_pattern="core.%e.%p"
          ulimit -c unlimited
          archery docker run ${{ matrix.image }}
      - name: Docker Push
        if: success() && github.event_name == 'push' && github.repository == 'apache/arrow'
        continue-on-error: true
        run: archery docker push ${{ matrix.image }}