From 1590bae82a7cd2c68244207cf0995631a689c8e6 Mon Sep 17 00:00:00 2001 From: mauroebordon Date: Thu, 13 Oct 2022 14:30:25 -0300 Subject: [PATCH 01/39] WIP github migration --- .github/workflows/dev.workflow.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/workflows/dev.workflow.yml diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml new file mode 100644 index 0000000..0ffdd02 --- /dev/null +++ b/.github/workflows/dev.workflow.yml @@ -0,0 +1 @@ +// TODO \ No newline at end of file From 0197de4241feddf0a4875e0c29aae896ab3aeef1 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Fri, 14 Oct 2022 16:37:48 -0300 Subject: [PATCH 02/39] parcial migration of the first 2 stages --- .github/workflows/dev.workflow.yml | 104 ++++++++++++++++++++++++++++- 1 file changed, 103 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 0ffdd02..55cfc64 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -1 +1,103 @@ -// TODO \ No newline at end of file +name: Integration workflow - validation & testing + +run-name: ${{github.repository}} is getting updated by ${{github.actor}} + +on: + pull_request: + branches: + - main + - master + +env: + PIP_CACHE_DIR: ${{github.workspace}}/.cache/pip + PRE_COMMIT_HOME: ${{github.workspace}}/.cache/pre-commit + POSTGRES_DB: soam_db + POSTGRES_USER: soam_usr + POSTGRES_PASSWORD: soam_pass + TEST_DB_CONNSTR: "postgresql://soam_usr:soam_pass@postgres:5432/soam_db" + +jobs: + prev_steps: + runs-on: ubuntu-latest + container: python:3.8.5 + steps: + - name: Cache Nox + uses: actions/cache@v3 + with: + path: | + .nox/ + .cache/pip + apt-cache/ + ${PRE_COMMIT_HOME} + + - name: Running Configuration + run: | + export APT_CACHE_DIR=`pwd`/apt-cache && mkdir -pv $APT_CACHE_DIR + apt-get update -yq && apt-get -o dir::cache::archives="$APT_CACHE_DIR" upgrade -y + apt-get install texlive-xetex texlive-fonts-recommended libpoppler-cpp-dev -y + + - name: Installing Nox + run: pip install nox + + + + +# stages: +# - validate +# - test_stage + + +# check_version: +# stage: validate +# script: +# - pip install packaging +# - git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME +# - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME +# - lib_ver=$(git diff origin/$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME -- soam/__init__.py | grep __version__ | cut -d = -f 2 | xargs) +# - python -c "import sys; from packaging import version; exit(not version.parse(sys.argv[1]) > version.parse(sys.argv[2]))" $lib_ver +# - exit_status=$? +# - if [ $exit_status -eq 1 ]; then echo "Error comparing versions"; fi; +# - exit $exit_status +# only: +# refs: +# - merge_requests +# variables: +# - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master" + +# check_changelog: +# stage: validate +# script: +# - git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME +# - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME +# - added_lines=$(git diff --numstat origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME origin/$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME -- CHANGELOG.md | awk '{print $1}') +# - if [ -z $added_lines ] || [ $added_lines -eq 0 ]; then echo "Changelog has not been modified" && exit 1; else echo "Changelog has been modified" && exit 0; fi; +# only: +# refs: +# - merge_requests +# variables: +# - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master" + + tests: + needs: [check_version, check_changelog] + steps: + - name: Tests + run: nox --sessions tests + + - name: Lint + run: nox --sessions lint + + - name: DocString Coverage + run: | + pip install interrogate + interrogate soam -c pyproject.toml -vv + + - name: Bandit + run: nox --sessions bandit + + - name: Pyreverse + run: | + apt-get -qq update + apt-get -qq install -y graphviz + nox --sessions pyreverse + + From 71a5b2d505714d1b86df36d65b8b1aa19a41c98f Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Fri, 14 Oct 2022 17:00:02 -0300 Subject: [PATCH 03/39] first draft ready for testing with act --- .github/workflows/dev.workflow.yml | 65 ++++++++++++------------------ 1 file changed, 25 insertions(+), 40 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 55cfc64..5dbf2c1 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -17,7 +17,7 @@ env: TEST_DB_CONNSTR: "postgresql://soam_usr:soam_pass@postgres:5432/soam_db" jobs: - prev_steps: + setup: runs-on: ubuntu-latest container: python:3.8.5 steps: @@ -39,46 +39,31 @@ jobs: - name: Installing Nox run: pip install nox - - - -# stages: -# - validate -# - test_stage - - -# check_version: -# stage: validate -# script: -# - pip install packaging -# - git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME -# - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -# - lib_ver=$(git diff origin/$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME -- soam/__init__.py | grep __version__ | cut -d = -f 2 | xargs) -# - python -c "import sys; from packaging import version; exit(not version.parse(sys.argv[1]) > version.parse(sys.argv[2]))" $lib_ver -# - exit_status=$? -# - if [ $exit_status -eq 1 ]; then echo "Error comparing versions"; fi; -# - exit $exit_status -# only: -# refs: -# - merge_requests -# variables: -# - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master" - -# check_changelog: -# stage: validate -# script: -# - git fetch origin $CI_MERGE_REQUEST_SOURCE_BRANCH_NAME -# - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME -# - added_lines=$(git diff --numstat origin/$CI_MERGE_REQUEST_TARGET_BRANCH_NAME origin/$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME -- CHANGELOG.md | awk '{print $1}') -# - if [ -z $added_lines ] || [ $added_lines -eq 0 ]; then echo "Changelog has not been modified" && exit 1; else echo "Changelog has been modified" && exit 0; fi; -# only: -# refs: -# - merge_requests -# variables: -# - $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master" + validate: + runs-on: ubuntu-latest + needs: [setup] + steps: + - name: Check version + run: | + pip install packaging + git fetch origin ${{github.head_ref}} + git fetch origin ${{github.base_ref}} + lib_ber=$(git diff origin/${{github.head_ref}}/${{github.base_ref}}) + python -c "import sys; prom packaging import version; exit(not version.parse(sys.argv[1]) > version.parse(sys.argv[2]))" $lib_ver + exit_status=$? + if [ $exit_status -eq 1 ]; then echo "Error comparing versions"; fi; + exit $exit_status + + - name: Check changelog + run: | + git fetch origin ${{github.head_ref}} + git fetch origin ${{github.base_ref}} + added_lines=$(git diff --numstat origin/${{github.base_ref}} origin/${{github.head_ref}} -- CHANGELOG.md | awk '{print $1}') + if [ -z $added_lines ] || [ $added_lines -eq 0 ]; then echo "Changelog has not been modified" && exit 1; else echo "Changelog has been modified" && exit 0; fi; - tests: - needs: [check_version, check_changelog] + test_stage: + runs-on: ubuntu-latest + needs: [validate] steps: - name: Tests run: nox --sessions tests From 2d8f1fe4ddc2d5c55281a368d139cf939317d049 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Mon, 17 Oct 2022 15:28:50 -0300 Subject: [PATCH 04/39] checking if error message is the same as in Act debugger --- .github/workflows/dev.workflow.yml | 39 +++++++++++------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 5dbf2c1..99a1f7b 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -1,8 +1,14 @@ +#act pull_request -j build-deploy + name: Integration workflow - validation & testing run-name: ${{github.repository}} is getting updated by ${{github.actor}} on: + push: + branches: + - main + - master pull_request: branches: - main @@ -17,31 +23,10 @@ env: TEST_DB_CONNSTR: "postgresql://soam_usr:soam_pass@postgres:5432/soam_db" jobs: - setup: - runs-on: ubuntu-latest - container: python:3.8.5 - steps: - - name: Cache Nox - uses: actions/cache@v3 - with: - path: | - .nox/ - .cache/pip - apt-cache/ - ${PRE_COMMIT_HOME} - - - name: Running Configuration - run: | - export APT_CACHE_DIR=`pwd`/apt-cache && mkdir -pv $APT_CACHE_DIR - apt-get update -yq && apt-get -o dir::cache::archives="$APT_CACHE_DIR" upgrade -y - apt-get install texlive-xetex texlive-fonts-recommended libpoppler-cpp-dev -y - - - name: Installing Nox - run: pip install nox - validate: runs-on: ubuntu-latest - needs: [setup] + container: python:3.8.5 + if: github.event_name == 'pull_request' steps: - name: Check version run: | @@ -53,7 +38,7 @@ jobs: exit_status=$? if [ $exit_status -eq 1 ]; then echo "Error comparing versions"; fi; exit $exit_status - + - name: Check changelog run: | git fetch origin ${{github.head_ref}} @@ -63,8 +48,12 @@ jobs: test_stage: runs-on: ubuntu-latest - needs: [validate] + container: python:3.8.5 steps: + - uses: actions/checkout@v3 + - name: Installing Nox + run: pip install -U nox + - name: Tests run: nox --sessions tests From e2bc8d28341783955242d214e651c0ee2f4aaae3 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 14:22:49 -0300 Subject: [PATCH 05/39] add checkout capability and installing pdftotext before faulty setup --- .github/workflows/dev.workflow.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 99a1f7b..d0b5177 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -27,13 +27,17 @@ jobs: runs-on: ubuntu-latest container: python:3.8.5 if: github.event_name == 'pull_request' + env: + TARGET_BRANCH: ${{ github.base_ref }} + HEAD_BRANCH: ${{ github.head_ref }} steps: + - uses: actions/checkout@v3 - name: Check version run: | pip install packaging - git fetch origin ${{github.head_ref}} - git fetch origin ${{github.base_ref}} - lib_ber=$(git diff origin/${{github.head_ref}}/${{github.base_ref}}) + git fetch origin HEAD_BRANCH + git fetch origin TARGET_BRANCH + lib_ber=$(git diff origin/HEAD_BRANCH/TARGET_BRANCH) python -c "import sys; prom packaging import version; exit(not version.parse(sys.argv[1]) > version.parse(sys.argv[2]))" $lib_ver exit_status=$? if [ $exit_status -eq 1 ]; then echo "Error comparing versions"; fi; @@ -41,9 +45,9 @@ jobs: - name: Check changelog run: | - git fetch origin ${{github.head_ref}} - git fetch origin ${{github.base_ref}} - added_lines=$(git diff --numstat origin/${{github.base_ref}} origin/${{github.head_ref}} -- CHANGELOG.md | awk '{print $1}') + git fetch origin HEAD_BRANCH + git fetch origin TARGET_BRANCH + added_lines=$(git diff --numstat origin/TARGET_BRANCH origin/HEAD_BRANCH -- CHANGELOG.md | awk '{print $1}') if [ -z $added_lines ] || [ $added_lines -eq 0 ]; then echo "Changelog has not been modified" && exit 1; else echo "Changelog has been modified" && exit 0; fi; test_stage: @@ -55,7 +59,9 @@ jobs: run: pip install -U nox - name: Tests - run: nox --sessions tests + run: | + pip install pdftotext + nox --sessions tests - name: Lint run: nox --sessions lint From 5be03e4d3598e0bc89f5e129ec22f53f4d4b7fee Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 14:40:46 -0300 Subject: [PATCH 06/39] Ref fix & installing poppler dev library with super user --- .github/workflows/dev.workflow.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index d0b5177..00d8406 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -35,9 +35,9 @@ jobs: - name: Check version run: | pip install packaging - git fetch origin HEAD_BRANCH - git fetch origin TARGET_BRANCH - lib_ber=$(git diff origin/HEAD_BRANCH/TARGET_BRANCH) + git fetch origin $HEAD_BRANCH + git fetch origin $TARGET_BRANCH + lib_ber=$(git diff origin/$HEAD_BRANCH/$TARGET_BRANCH) python -c "import sys; prom packaging import version; exit(not version.parse(sys.argv[1]) > version.parse(sys.argv[2]))" $lib_ver exit_status=$? if [ $exit_status -eq 1 ]; then echo "Error comparing versions"; fi; @@ -45,14 +45,16 @@ jobs: - name: Check changelog run: | - git fetch origin HEAD_BRANCH - git fetch origin TARGET_BRANCH - added_lines=$(git diff --numstat origin/TARGET_BRANCH origin/HEAD_BRANCH -- CHANGELOG.md | awk '{print $1}') + git fetch origin $HEAD_BRANCH + git fetch origin $TARGET_BRANCH + added_lines=$(git diff --numstat origin/$TARGET_BRANCH origin/$HEAD_BRANCH -- CHANGELOG.md | awk '{print $1}') if [ -z $added_lines ] || [ $added_lines -eq 0 ]; then echo "Changelog has not been modified" && exit 1; else echo "Changelog has been modified" && exit 0; fi; test_stage: runs-on: ubuntu-latest - container: python:3.8.5 + container: + image: python:3.8.5 + options: --user root steps: - uses: actions/checkout@v3 - name: Installing Nox @@ -60,6 +62,9 @@ jobs: - name: Tests run: | + sudo add-apt-repository -y ppa:cran/poppler + sudo apt-get update + sudo apt-get install -y libpoppler-cpp-dev libpoppler-dev poppler-utils pip install pdftotext nox --sessions tests From 4db3acd91c450efcecb03f0598b5a738540c8072 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 14:49:03 -0300 Subject: [PATCH 07/39] More fixes --- .github/workflows/dev.workflow.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 00d8406..1963c3a 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -32,12 +32,15 @@ jobs: HEAD_BRANCH: ${{ github.head_ref }} steps: - uses: actions/checkout@v3 + - name: Installing sudo package + run: apt update && apt install sudo + - name: Check version run: | pip install packaging git fetch origin $HEAD_BRANCH git fetch origin $TARGET_BRANCH - lib_ber=$(git diff origin/$HEAD_BRANCH/$TARGET_BRANCH) + lib_ber=$(git diff origin/$HEAD_BRANCH origin/$TARGET_BRANCH -- soam/__init__.py | grep __version__ | cut -d = -f 2 | xargs) python -c "import sys; prom packaging import version; exit(not version.parse(sys.argv[1]) > version.parse(sys.argv[2]))" $lib_ver exit_status=$? if [ $exit_status -eq 1 ]; then echo "Error comparing versions"; fi; From 838a86ee1b78870ae7670de54467f7a17f87b57c Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 14:51:35 -0300 Subject: [PATCH 08/39] More fixes --- .github/workflows/dev.workflow.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 1963c3a..af6eddb 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -32,16 +32,13 @@ jobs: HEAD_BRANCH: ${{ github.head_ref }} steps: - uses: actions/checkout@v3 - - name: Installing sudo package - run: apt update && apt install sudo - - name: Check version run: | pip install packaging git fetch origin $HEAD_BRANCH git fetch origin $TARGET_BRANCH lib_ber=$(git diff origin/$HEAD_BRANCH origin/$TARGET_BRANCH -- soam/__init__.py | grep __version__ | cut -d = -f 2 | xargs) - python -c "import sys; prom packaging import version; exit(not version.parse(sys.argv[1]) > version.parse(sys.argv[2]))" $lib_ver + python -c "import sys; from packaging import version; exit(not version.parse(sys.argv[1]) > version.parse(sys.argv[2]))" $lib_ver exit_status=$? if [ $exit_status -eq 1 ]; then echo "Error comparing versions"; fi; exit $exit_status @@ -60,6 +57,10 @@ jobs: options: --user root steps: - uses: actions/checkout@v3 + + - name: Installing sudo package + run: apt update && apt install sudo + - name: Installing Nox run: pip install -U nox From 3630709c470acacc763a48b2654665aa3ea99943 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 15:01:30 -0300 Subject: [PATCH 09/39] advancing version to be able to test the validation stage correctly --- soam/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/soam/__init__.py b/soam/__init__.py index e7ade49..85a9795 100644 --- a/soam/__init__.py +++ b/soam/__init__.py @@ -1,3 +1,3 @@ """Version.""" -__version__ = '0.10.1' +__version__ = '0.10.2' From dfa74ec354f5c4b71c876fa641cb3bc16e9ffeb6 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 15:06:53 -0300 Subject: [PATCH 10/39] fixes --- .github/workflows/dev.workflow.yml | 4 +--- CHANGELOG.md | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index af6eddb..cf27728 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -38,7 +38,7 @@ jobs: git fetch origin $HEAD_BRANCH git fetch origin $TARGET_BRANCH lib_ber=$(git diff origin/$HEAD_BRANCH origin/$TARGET_BRANCH -- soam/__init__.py | grep __version__ | cut -d = -f 2 | xargs) - python -c "import sys; from packaging import version; exit(not version.parse(sys.argv[1]) > version.parse(sys.argv[2]))" $lib_ver + python -c "import sys; from packaging import version; exit(not version.parse(sys.argv[0]) > version.parse(sys.argv[1]))" $lib_ver exit_status=$? if [ $exit_status -eq 1 ]; then echo "Error comparing versions"; fi; exit $exit_status @@ -66,8 +66,6 @@ jobs: - name: Tests run: | - sudo add-apt-repository -y ppa:cran/poppler - sudo apt-get update sudo apt-get install -y libpoppler-cpp-dev libpoppler-dev poppler-utils pip install pdftotext nox --sessions tests diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d2d6a9..a63c241 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.10.2- 2022-10-18] +- Added Github Workflow + ## [0.10.1- 2022-02-14] ### Fixed From c717fb3c30920aa0686ca6142ba4c93f4677b68d Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 15:09:42 -0300 Subject: [PATCH 11/39] sintax error --- .github/workflows/dev.workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index cf27728..9b45dc3 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -37,7 +37,7 @@ jobs: pip install packaging git fetch origin $HEAD_BRANCH git fetch origin $TARGET_BRANCH - lib_ber=$(git diff origin/$HEAD_BRANCH origin/$TARGET_BRANCH -- soam/__init__.py | grep __version__ | cut -d = -f 2 | xargs) + lib_ver=$(git diff origin/$HEAD_BRANCH origin/$TARGET_BRANCH -- soam/__init__.py | grep __version__ | cut -d = -f 2 | xargs) python -c "import sys; from packaging import version; exit(not version.parse(sys.argv[0]) > version.parse(sys.argv[1]))" $lib_ver exit_status=$? if [ $exit_status -eq 1 ]; then echo "Error comparing versions"; fi; From c468371f0ed4b4bb1d2ba1ba511b07014c500286 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 15:22:14 -0300 Subject: [PATCH 12/39] the script is working locally --- .github/workflows/dev.workflow.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 9b45dc3..9cbba3e 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -32,16 +32,13 @@ jobs: HEAD_BRANCH: ${{ github.head_ref }} steps: - uses: actions/checkout@v3 - - name: Check version + - name: Comparing if new version run: | pip install packaging git fetch origin $HEAD_BRANCH git fetch origin $TARGET_BRANCH lib_ver=$(git diff origin/$HEAD_BRANCH origin/$TARGET_BRANCH -- soam/__init__.py | grep __version__ | cut -d = -f 2 | xargs) - python -c "import sys; from packaging import version; exit(not version.parse(sys.argv[0]) > version.parse(sys.argv[1]))" $lib_ver - exit_status=$? - if [ $exit_status -eq 1 ]; then echo "Error comparing versions"; fi; - exit $exit_status + python -c "import sys; from packaging import version; v1, v2 = sys.argv[1].split(); exit(not version.parse(v1) > version.parse(v2))" $lib_ver - name: Check changelog run: | @@ -52,6 +49,7 @@ jobs: test_stage: runs-on: ubuntu-latest + continue-on-error: true container: image: python:3.8.5 options: --user root From 52c40ce65049a650695fd5f0833ec538f577fcfa Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 15:27:58 -0300 Subject: [PATCH 13/39] debbuging --- .github/workflows/dev.workflow.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 9cbba3e..14cc886 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -38,8 +38,9 @@ jobs: git fetch origin $HEAD_BRANCH git fetch origin $TARGET_BRANCH lib_ver=$(git diff origin/$HEAD_BRANCH origin/$TARGET_BRANCH -- soam/__init__.py | grep __version__ | cut -d = -f 2 | xargs) - python -c "import sys; from packaging import version; v1, v2 = sys.argv[1].split(); exit(not version.parse(v1) > version.parse(v2))" $lib_ver - + python -c "import sys; from packaging import version; print(sys.argv[1])" $lib_ver + echo $lib_ver + - name: Check changelog run: | git fetch origin $HEAD_BRANCH From 6e76dd67d226187d787667354153e3e15e7ac1bf Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 15:31:02 -0300 Subject: [PATCH 14/39] debbuging --- .github/workflows/dev.workflow.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 14cc886..456820a 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -32,15 +32,14 @@ jobs: HEAD_BRANCH: ${{ github.head_ref }} steps: - uses: actions/checkout@v3 - - name: Comparing if new version + - name: New Version Check run: | pip install packaging git fetch origin $HEAD_BRANCH git fetch origin $TARGET_BRANCH lib_ver=$(git diff origin/$HEAD_BRANCH origin/$TARGET_BRANCH -- soam/__init__.py | grep __version__ | cut -d = -f 2 | xargs) - python -c "import sys; from packaging import version; print(sys.argv[1])" $lib_ver - echo $lib_ver - + python -c "import sys; from packaging import version; exit(not version.parse(sys.argv[1]) > version.parse(sys.argv[2]))" $lib_ver + - name: Check changelog run: | git fetch origin $HEAD_BRANCH From 24629e9b158185b09adadb5493a3bb55b230efb6 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 15:32:26 -0300 Subject: [PATCH 15/39] skipping tests --- .github/workflows/dev.workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 456820a..9639599 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -49,7 +49,6 @@ jobs: test_stage: runs-on: ubuntu-latest - continue-on-error: true container: image: python:3.8.5 options: --user root @@ -63,6 +62,7 @@ jobs: run: pip install -U nox - name: Tests + continue-on-error: true run: | sudo apt-get install -y libpoppler-cpp-dev libpoppler-dev poppler-utils pip install pdftotext From e74f4f9d4abfd1d41e7183bfa6a07129d27337d1 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 16:14:22 -0300 Subject: [PATCH 16/39] checking all steps --- .github/workflows/dev.workflow.yml | 43 +++++++++++++++++++----------- .pre-commit-config.yaml | 4 +-- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 9639599..2f07465 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -1,14 +1,10 @@ -#act pull_request -j build-deploy +#act pull_request -j build-deploy name: Integration workflow - validation & testing -run-name: ${{github.repository}} is getting updated by ${{github.actor}} +run-name: ${{github.repository}} is getting updated by ${{github.actor}} on: - push: - branches: - - main - - master pull_request: branches: - main @@ -26,7 +22,7 @@ jobs: validate: runs-on: ubuntu-latest container: python:3.8.5 - if: github.event_name == 'pull_request' + #if: github.event_name == 'pull_request' env: TARGET_BRANCH: ${{ github.base_ref }} HEAD_BRANCH: ${{ github.head_ref }} @@ -37,13 +33,13 @@ jobs: pip install packaging git fetch origin $HEAD_BRANCH git fetch origin $TARGET_BRANCH - lib_ver=$(git diff origin/$HEAD_BRANCH origin/$TARGET_BRANCH -- soam/__init__.py | grep __version__ | cut -d = -f 2 | xargs) + lib_ver=$(git diff origin/$HEAD_BRANCH origin/$TARGET_BRANCH -- soam/__init__.py | grep __version__ | cut -d = -f 2 | xargs) python -c "import sys; from packaging import version; exit(not version.parse(sys.argv[1]) > version.parse(sys.argv[2]))" $lib_ver - name: Check changelog run: | git fetch origin $HEAD_BRANCH - git fetch origin $TARGET_BRANCH + git fetch origin $TARGET_BRANCH added_lines=$(git diff --numstat origin/$TARGET_BRANCH origin/$HEAD_BRANCH -- CHANGELOG.md | awk '{print $1}') if [ -z $added_lines ] || [ $added_lines -eq 0 ]; then echo "Changelog has not been modified" && exit 1; else echo "Changelog has been modified" && exit 0; fi; @@ -57,9 +53,9 @@ jobs: - name: Installing sudo package run: apt update && apt install sudo - + - name: Installing Nox - run: pip install -U nox + run: pip install -U nox - name: Tests continue-on-error: true @@ -67,22 +63,39 @@ jobs: sudo apt-get install -y libpoppler-cpp-dev libpoppler-dev poppler-utils pip install pdftotext nox --sessions tests - + - name: Lint + continue-on-error: true run: nox --sessions lint - + - name: DocString Coverage + continue-on-error: true run: | pip install interrogate interrogate soam -c pyproject.toml -vv - + - name: Bandit + continue-on-error: true run: nox --sessions bandit - + - name: Pyreverse run: | apt-get -qq update apt-get -qq install -y graphviz nox --sessions pyreverse + # version_stage: + # runs-on: ubuntu-latest + # container: python:3.8.5 + # steps: + # script: + # - echo "TAG=v$(grep __version__ ./soam/__init__.py | cut -d "'" -f 2)" >> variables.env + # artifacts: + # reports: + # dotenv: variables.env + # before_script: + # - '' + # only: + # refs: + # - master \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 66c2147..3ae9661 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: templates/.+|notebook/.+ repos: -- repo: git://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 +- repo: https://github.com/pre-commit/pre-commit-hooks.git + rev: v4.3.0 hooks: - id: trailing-whitespace - id: requirements-txt-fixer From f6fea3e24137ee7788940cc6c375c5c213ca2270 Mon Sep 17 00:00:00 2001 From: a-rusi Date: Tue, 18 Oct 2022 17:37:37 -0300 Subject: [PATCH 17/39] change in failing dependencies --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index a45f8d2..8ab633c 100644 --- a/setup.py +++ b/setup.py @@ -43,9 +43,10 @@ 'slack': ["slackclient>=2.0.0"], 'orbit': ['orbit-ml==1.0.13'], 'prophet': ["pystan==2.19.1.1", "fbprophet==0.7.1", "holidays>=0.10.2"], - 'pdf_report': ["jupytext==1.10.2", "papermill==2.3.2", "nbconvert==5.6",], + 'pdf_report': ["jupytext==1.10.2", "papermill==2.3.2", "nbconvert==5.6","ipython_genutils"], 'gsheets_report': ["gspread_pandas", "muttlib[gsheets]>=1.0,<2"], 'statsmodels': ["statsmodels<0.12,>=0.11"], + 'protobuf': ["protobuf~=3.2"], 'mlflow': ["mlflow==1.17.0"], } @@ -84,7 +85,7 @@ tests_require=["pytest", "pytest-cov", "pytest-html", "betamax"], test_suite='test', install_requires=[ - "jinja2", + "Jinja2>=2.11.3,<3.1.0", "pandas>=1.0.0,<1.3.0", "Cython<0.29.18,>=0.29", "sqlalchemy<1.4.0,>=1.3.0", From 2e726d77145ab326dc163f8341566947624ad22c Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 17:40:46 -0300 Subject: [PATCH 18/39] working on testing coverage --- .github/workflows/dev.workflow.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 2f07465..9dc2979 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -48,6 +48,14 @@ jobs: container: image: python:3.8.5 options: --user root + services: + postgres: + image: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - uses: actions/checkout@v3 @@ -58,24 +66,24 @@ jobs: run: pip install -U nox - name: Tests - continue-on-error: true + #continue-on-error: true run: | sudo apt-get install -y libpoppler-cpp-dev libpoppler-dev poppler-utils pip install pdftotext nox --sessions tests - name: Lint - continue-on-error: true + #continue-on-error: true run: nox --sessions lint - name: DocString Coverage - continue-on-error: true + #continue-on-error: true run: | pip install interrogate interrogate soam -c pyproject.toml -vv - name: Bandit - continue-on-error: true + #continue-on-error: true run: nox --sessions bandit - name: Pyreverse From 3cb2108cad836b087373de7b967fddac8bf6de11 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 17:44:54 -0300 Subject: [PATCH 19/39] postgres service fix --- .github/workflows/dev.workflow.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 9dc2979..ba2ef02 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -13,10 +13,6 @@ on: env: PIP_CACHE_DIR: ${{github.workspace}}/.cache/pip PRE_COMMIT_HOME: ${{github.workspace}}/.cache/pre-commit - POSTGRES_DB: soam_db - POSTGRES_USER: soam_usr - POSTGRES_PASSWORD: soam_pass - TEST_DB_CONNSTR: "postgresql://soam_usr:soam_pass@postgres:5432/soam_db" jobs: validate: @@ -51,6 +47,8 @@ jobs: services: postgres: image: postgres + env: + POSTGRES_PASSWORD: soam_pass options: >- --health-cmd pg_isready --health-interval 10s @@ -67,6 +65,10 @@ jobs: - name: Tests #continue-on-error: true + env: + POSTGRES_DB: soam_db + POSTGRES_USER: soam_usr + TEST_DB_CONNSTR: "postgresql://soam_usr:soam_pass@postgres:5432/soam_db" run: | sudo apt-get install -y libpoppler-cpp-dev libpoppler-dev poppler-utils pip install pdftotext From e34546688facdb90b4fd75d367ebe0f5881b1f8e Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 18:20:20 -0300 Subject: [PATCH 20/39] postgres service fix --- .github/workflows/dev.workflow.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index ba2ef02..02f3950 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -66,6 +66,8 @@ jobs: - name: Tests #continue-on-error: true env: + POSTGRES_HOST: localhost + POSTGRES_PORT: 5432 POSTGRES_DB: soam_db POSTGRES_USER: soam_usr TEST_DB_CONNSTR: "postgresql://soam_usr:soam_pass@postgres:5432/soam_db" From e15434cf262858bb59c0bbc6cc6a9df266c4b2a7 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 18:52:08 -0300 Subject: [PATCH 21/39] lint & tests fix --- .github/workflows/dev.workflow.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 02f3950..ced5236 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -48,7 +48,12 @@ jobs: postgres: image: postgres env: + POSTGRES_DB: soam_db + POSTGRES_USER: soam_usr POSTGRES_PASSWORD: soam_pass + ports: + + - 5432:5432 options: >- --health-cmd pg_isready --health-interval 10s @@ -64,13 +69,11 @@ jobs: run: pip install -U nox - name: Tests - #continue-on-error: true + continue-on-error: true env: POSTGRES_HOST: localhost POSTGRES_PORT: 5432 - POSTGRES_DB: soam_db - POSTGRES_USER: soam_usr - TEST_DB_CONNSTR: "postgresql://soam_usr:soam_pass@postgres:5432/soam_db" + # TEST_DB_CONNSTR: "postgresql://soam_usr:soam_pass@postgres:5432/soam_db" run: | sudo apt-get install -y libpoppler-cpp-dev libpoppler-dev poppler-utils pip install pdftotext From 4d13f73db4586751f18d99e3d117d6d82d43093b Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 18:57:41 -0300 Subject: [PATCH 22/39] syntax fix --- .github/workflows/dev.workflow.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index ced5236..e6253c7 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -51,9 +51,7 @@ jobs: POSTGRES_DB: soam_db POSTGRES_USER: soam_usr POSTGRES_PASSWORD: soam_pass - ports: - - - 5432:5432 + ports: 5432:5432 options: >- --health-cmd pg_isready --health-interval 10s From aedcef54904d128a67cee9e71e7d086e83461791 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 19:03:57 -0300 Subject: [PATCH 23/39] syntax fix --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3ae9661..f9608c5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: # args: [--max-line-length=88] # exclude: templates/* notebook/* - repo: https://github.com/timothycrosley/isort - rev: 4.3.21 + rev: 5.10.1 hooks: - id: isort name: isort @@ -22,7 +22,7 @@ repos: types: [python] exclude: templates/.+|notebook/.+ - repo: https://github.com/psf/black - rev: 19.10b0 + rev: 22.10.0 hooks: - id: black name: black @@ -32,7 +32,7 @@ repos: args: [--skip-string-normalization] exclude: templates/.+|notebook/.+ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.761 + rev: v0.982 hooks: - id: mypy name: mypy @@ -42,7 +42,7 @@ repos: files: \.py$ exclude: templates/.+|notebook/.+ - repo: https://github.com/pre-commit/mirrors-pylint - rev: v2.4.4 + rev: v3.0.0a5 hooks: - id: pylint name: pylint From 9ac93a23bf3ada41bbd49fbfe72129a1a84dc5cf Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Tue, 18 Oct 2022 19:05:12 -0300 Subject: [PATCH 24/39] syntax fix --- .github/workflows/dev.workflow.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index e6253c7..e867412 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -51,7 +51,8 @@ jobs: POSTGRES_DB: soam_db POSTGRES_USER: soam_usr POSTGRES_PASSWORD: soam_pass - ports: 5432:5432 + ports: + - 5432:5432 options: >- --health-cmd pg_isready --health-interval 10s From 34e9a0f8e44abd25790cd8bef3bc6afc67791533 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Wed, 19 Oct 2022 15:50:39 -0300 Subject: [PATCH 25/39] fixing precommit error while running lint --- .github/workflows/dev.workflow.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index e867412..7ddc888 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -80,7 +80,9 @@ jobs: - name: Lint #continue-on-error: true - run: nox --sessions lint + run: | + pip install --upgrade pre-commit + nox --sessions lint - name: DocString Coverage #continue-on-error: true From 904cf6677289537609ec60c79bd33aed26488e00 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Thu, 20 Oct 2022 14:24:18 -0300 Subject: [PATCH 26/39] testing with old pre-commits configuration --- .github/workflows/dev.workflow.yml | 23 +++++++++- .pre-commit-config.yaml | 68 +++++++++++++++++++++++++++--- 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 7ddc888..29a1082 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -79,7 +79,7 @@ jobs: nox --sessions tests - name: Lint - #continue-on-error: true + continue-on-error: true run: | pip install --upgrade pre-commit nox --sessions lint @@ -114,4 +114,23 @@ jobs: # - '' # only: # refs: - # - master \ No newline at end of file + # - master + + release_stage: + name: Release Stage + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # This token is provided by Actions, you do not need to create your own token + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + body: | + Adding Github Workflows + draft: false + prerelease: false \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f9608c5..ea45254 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: templates/.+|notebook/.+ repos: -- repo: https://github.com/pre-commit/pre-commit-hooks.git - rev: v4.3.0 +- repo: git://github.com/pre-commit/pre-commit-hooks + rev: v2.5.0 hooks: - id: trailing-whitespace - id: requirements-txt-fixer @@ -13,7 +13,7 @@ repos: # args: [--max-line-length=88] # exclude: templates/* notebook/* - repo: https://github.com/timothycrosley/isort - rev: 5.10.1 + rev: 4.3.21 hooks: - id: isort name: isort @@ -22,7 +22,7 @@ repos: types: [python] exclude: templates/.+|notebook/.+ - repo: https://github.com/psf/black - rev: 22.10.0 + rev: 19.10b0 hooks: - id: black name: black @@ -32,7 +32,7 @@ repos: args: [--skip-string-normalization] exclude: templates/.+|notebook/.+ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.982 + rev: v0.761 hooks: - id: mypy name: mypy @@ -42,7 +42,7 @@ repos: files: \.py$ exclude: templates/.+|notebook/.+ - repo: https://github.com/pre-commit/mirrors-pylint - rev: v3.0.0a5 + rev: v2.4.4 hooks: - id: pylint name: pylint @@ -60,3 +60,59 @@ repos: types: [file] files: 'README.md' require_serial: true + +# exclude: templates/.+|notebook/.+ +# repos: +# - repo: https://github.com/pre-commit/pre-commit-hooks.git +# rev: v4.3.0 +# hooks: +# - id: trailing-whitespace +# - id: requirements-txt-fixer +# - repo: https://github.com/timothycrosley/isort +# rev: 5.10.1 +# hooks: +# - id: isort +# name: isort +# entry: isort +# language: system +# types: [python] +# exclude: templates/.+|notebook/.+ +# - repo: https://github.com/psf/black +# rev: 22.10.0 +# hooks: +# - id: black +# name: black +# entry: black +# language: python +# types: [python] +# args: [--skip-string-normalization] +# exclude: templates/.+|notebook/.+ +# - repo: https://github.com/pre-commit/mirrors-mypy +# rev: v0.982 +# hooks: +# - id: mypy +# name: mypy +# entry: mypy +# language: system +# types: [python] +# files: \.py$ +# exclude: templates/.+|notebook/.+ +# - repo: https://github.com/pre-commit/mirrors-pylint +# rev: v3.0.0a5 +# hooks: +# - id: pylint +# name: pylint +# entry: pylint +# language: system +# exclude: templates/.+|notebook/.+ +# files: \.py$ +# types: [python] +# - repo: https://github.com/MuttData/gfm-diagram +# rev: v0.3.0 +# hooks: +# - id: gfmd +# name: gfmd +# entry: gfmd +# types: [file] +# files: 'README.md' +# require_serial: true From c0e84a7de261c25fdda553eed919ff2bd4271821 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Thu, 20 Oct 2022 15:12:29 -0300 Subject: [PATCH 27/39] test_db_conn added --- .github/workflows/dev.workflow.yml | 40 ++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 29a1082..8cd2d63 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -41,6 +41,7 @@ jobs: test_stage: runs-on: ubuntu-latest + needs: validate container: image: python:3.8.5 options: --user root @@ -72,7 +73,7 @@ jobs: env: POSTGRES_HOST: localhost POSTGRES_PORT: 5432 - # TEST_DB_CONNSTR: "postgresql://soam_usr:soam_pass@postgres:5432/soam_db" + TEST_DB_CONNSTR: "postgresql://soam_usr:soam_pass@postgres:5432/soam_db" run: | sudo apt-get install -y libpoppler-cpp-dev libpoppler-dev poppler-utils pip install pdftotext @@ -119,6 +120,7 @@ jobs: release_stage: name: Release Stage runs-on: ubuntu-latest + needs: test_stage steps: - name: Checkout code uses: actions/checkout@v2 @@ -133,4 +135,38 @@ jobs: body: | Adding Github Workflows draft: false - prerelease: false \ No newline at end of file + prerelease: false + + # pages: + # stage: deploy_stage + # script: + # - pip install .[dev] + # - cd documentation + # - sphinx-apidoc -f -o source ../soam + # - make html + # - cd .. + # - mv documentation/build/html/ public/ + # artifacts: + # paths: + # - public + # only: + # refs: + # - master + + # Deploy from master to package registry +# If PYPI_USERNAME/PYPI_PASSWORD are not set, defaults to gitlab +# package registry, thus using the required variables to do so +# The package will be uploaded to PYPI_URL, which can be overrided +# not to upload to gitlab's PYPI +# deploy: +# stage: deploy_stage +# script: +# - pip install twine +# - python setup.py sdist bdist_wheel +# - export TWINE_USERNAME=${PYPI_USERNAME:=gitlab-ci-token} +# - export TWINE_PASSWORD=${PYPI_PASSWORD:=$CI_JOB_TOKEN} +# - export PYPI_REPO=${PYPI_URL:=https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/packages/pypi} +# - python -m twine upload --verbose --repository-url ${PYPI_REPO} dist/* +# only: +# refs: +# - master \ No newline at end of file From d540ea3866f62f44744752f4baec216a2429c228 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Thu, 20 Oct 2022 15:23:03 -0300 Subject: [PATCH 28/39] installing latex --- .github/workflows/dev.workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 8cd2d63..45708c3 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -75,7 +75,7 @@ jobs: POSTGRES_PORT: 5432 TEST_DB_CONNSTR: "postgresql://soam_usr:soam_pass@postgres:5432/soam_db" run: | - sudo apt-get install -y libpoppler-cpp-dev libpoppler-dev poppler-utils + sudo apt-get install -y texlive-xetex texlive-fonts-recommended libpoppler-cpp-dev libpoppler-dev poppler-utils pip install pdftotext nox --sessions tests From 905b973e46c410cdbf5cf147371ffb3064d94858 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Thu, 20 Oct 2022 15:41:31 -0300 Subject: [PATCH 29/39] fixing lint --- .github/workflows/dev.workflow.yml | 16 +++++++++------- .pre-commit-config.yaml | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 45708c3..54660ef 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -68,23 +68,25 @@ jobs: - name: Installing Nox run: pip install -U nox + + + - name: Lint + continue-on-error: true + run: | + pip install --upgrade pre-commit + nox --sessions lint + - name: Tests continue-on-error: true env: POSTGRES_HOST: localhost POSTGRES_PORT: 5432 TEST_DB_CONNSTR: "postgresql://soam_usr:soam_pass@postgres:5432/soam_db" + run: | sudo apt-get install -y texlive-xetex texlive-fonts-recommended libpoppler-cpp-dev libpoppler-dev poppler-utils pip install pdftotext nox --sessions tests - - - name: Lint - continue-on-error: true - run: | - pip install --upgrade pre-commit - nox --sessions lint - - name: DocString Coverage #continue-on-error: true run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ea45254..ebb7551 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ exclude: templates/.+|notebook/.+ repos: - repo: git://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 + rev: v4.3.0 hooks: - id: trailing-whitespace - id: requirements-txt-fixer From 79c16edde43688a0d40a02ca8a5753ab86d5166a Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Thu, 20 Oct 2022 15:47:58 -0300 Subject: [PATCH 30/39] checking lint error --- .github/workflows/dev.workflow.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 54660ef..66fcb45 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -68,25 +68,24 @@ jobs: - name: Installing Nox run: pip install -U nox - - - - name: Lint - continue-on-error: true - run: | - pip install --upgrade pre-commit - nox --sessions lint - - name: Tests continue-on-error: true env: POSTGRES_HOST: localhost POSTGRES_PORT: 5432 TEST_DB_CONNSTR: "postgresql://soam_usr:soam_pass@postgres:5432/soam_db" - + run: | sudo apt-get install -y texlive-xetex texlive-fonts-recommended libpoppler-cpp-dev libpoppler-dev poppler-utils pip install pdftotext - nox --sessions tests + nox --sessions lint + + - name: Lint + continue-on-error: true + run: | + pip install --upgrade pre-commit + nox --sessions lint + - name: DocString Coverage #continue-on-error: true run: | From 786b1fed21e343f61962a53130e5ebcd1b9545eb Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Thu, 20 Oct 2022 16:26:04 -0300 Subject: [PATCH 31/39] precommit autoupdate --- .github/workflows/dev.workflow.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 66fcb45..403d3e7 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -2,6 +2,7 @@ name: Integration workflow - validation & testing +#tal persona is working in PR_NAME run-name: ${{github.repository}} is getting updated by ${{github.actor}} on: From 00567912b3935ffac50ed8ee10f623c675cfc73c Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Thu, 20 Oct 2022 16:29:00 -0300 Subject: [PATCH 32/39] fix --- .github/workflows/dev.workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 403d3e7..01c99d8 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -79,7 +79,7 @@ jobs: run: | sudo apt-get install -y texlive-xetex texlive-fonts-recommended libpoppler-cpp-dev libpoppler-dev poppler-utils pip install pdftotext - nox --sessions lint + nox --sessions test - name: Lint continue-on-error: true From c20d2fb66e4212de292aadda4c35d4d3bd0c40d4 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Thu, 20 Oct 2022 18:19:51 -0300 Subject: [PATCH 33/39] fix --- .github/workflows/dev.workflow.yml | 4 ---- .pre-commit-config.yaml | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 01c99d8..f398d5b 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -70,7 +70,6 @@ jobs: run: pip install -U nox - name: Tests - continue-on-error: true env: POSTGRES_HOST: localhost POSTGRES_PORT: 5432 @@ -82,19 +81,16 @@ jobs: nox --sessions test - name: Lint - continue-on-error: true run: | pip install --upgrade pre-commit nox --sessions lint - name: DocString Coverage - #continue-on-error: true run: | pip install interrogate interrogate soam -c pyproject.toml -vv - name: Bandit - #continue-on-error: true run: nox --sessions bandit - name: Pyreverse diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ebb7551..4c31056 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ exclude: templates/.+|notebook/.+ repos: -- repo: git://github.com/pre-commit/pre-commit-hooks +- repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 hooks: - id: trailing-whitespace From 42352799c89cf268f423410db1d51fa9d1ec55ae Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Thu, 20 Oct 2022 18:23:59 -0300 Subject: [PATCH 34/39] fix --- .github/workflows/dev.workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index f398d5b..ce3187c 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -78,7 +78,7 @@ jobs: run: | sudo apt-get install -y texlive-xetex texlive-fonts-recommended libpoppler-cpp-dev libpoppler-dev poppler-utils pip install pdftotext - nox --sessions test + nox --sessions tests - name: Lint run: | From 00321a72c63364aa1e81fdb9c010f564c2d122ea Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Wed, 26 Oct 2022 15:21:54 -0300 Subject: [PATCH 35/39] changing to self-holsted runners --- .github/workflows/dev.workflow.yml | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index ce3187c..bba7a32 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -1,9 +1,6 @@ -#act pull_request -j build-deploy - name: Integration workflow - validation & testing -#tal persona is working in PR_NAME -run-name: ${{github.repository}} is getting updated by ${{github.actor}} +run-name: ${{github.event_name}} by ${{github.actor}} on: pull_request: @@ -17,9 +14,8 @@ env: jobs: validate: - runs-on: ubuntu-latest + runs-on: [self-hosted] container: python:3.8.5 - #if: github.event_name == 'pull_request' env: TARGET_BRANCH: ${{ github.base_ref }} HEAD_BRANCH: ${{ github.head_ref }} @@ -41,7 +37,7 @@ jobs: if [ -z $added_lines ] || [ $added_lines -eq 0 ]; then echo "Changelog has not been modified" && exit 1; else echo "Changelog has been modified" && exit 0; fi; test_stage: - runs-on: ubuntu-latest + runs-on: [self-hosted] needs: validate container: image: python:3.8.5 @@ -53,8 +49,7 @@ jobs: POSTGRES_DB: soam_db POSTGRES_USER: soam_usr POSTGRES_PASSWORD: soam_pass - ports: - - 5432:5432 + ports: 5432:5432 options: >- --health-cmd pg_isready --health-interval 10s @@ -69,22 +64,21 @@ jobs: - name: Installing Nox run: pip install -U nox + - name: Lint + run: | + sudo apt-get install -y texlive-xetex texlive-fonts-recommended libpoppler-cpp-dev libpoppler-dev poppler-utils + pip install --upgrade pre-commit + nox --sessions lint + - name: Tests env: POSTGRES_HOST: localhost POSTGRES_PORT: 5432 TEST_DB_CONNSTR: "postgresql://soam_usr:soam_pass@postgres:5432/soam_db" - run: | - sudo apt-get install -y texlive-xetex texlive-fonts-recommended libpoppler-cpp-dev libpoppler-dev poppler-utils pip install pdftotext nox --sessions tests - - name: Lint - run: | - pip install --upgrade pre-commit - nox --sessions lint - - name: DocString Coverage run: | pip install interrogate @@ -100,7 +94,7 @@ jobs: nox --sessions pyreverse # version_stage: - # runs-on: ubuntu-latest + # runs-on: [self-hosted] # container: python:3.8.5 # steps: @@ -117,7 +111,7 @@ jobs: release_stage: name: Release Stage - runs-on: ubuntu-latest + runs-on: [self-hosted] needs: test_stage steps: - name: Checkout code From 7764206344e5f5d14d3e87164c6d1106369d5a59 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Wed, 26 Oct 2022 15:41:26 -0300 Subject: [PATCH 36/39] fix --- .github/workflows/dev.workflow.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index bba7a32..f228cc0 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -49,7 +49,7 @@ jobs: POSTGRES_DB: soam_db POSTGRES_USER: soam_usr POSTGRES_PASSWORD: soam_pass - ports: 5432:5432 + ports: 5432 options: >- --health-cmd pg_isready --health-interval 10s @@ -66,6 +66,7 @@ jobs: - name: Lint run: | + sudo apt-get install libffi-dev sudo apt-get install -y texlive-xetex texlive-fonts-recommended libpoppler-cpp-dev libpoppler-dev poppler-utils pip install --upgrade pre-commit nox --sessions lint From c70bb4d06059be0faf6d87c32d504756f22886c7 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Wed, 26 Oct 2022 15:44:46 -0300 Subject: [PATCH 37/39] fix --- .github/workflows/dev.workflow.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index f228cc0..81f25ae 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -49,7 +49,7 @@ jobs: POSTGRES_DB: soam_db POSTGRES_USER: soam_usr POSTGRES_PASSWORD: soam_pass - ports: 5432 + options: >- --health-cmd pg_isready --health-interval 10s @@ -71,6 +71,7 @@ jobs: pip install --upgrade pre-commit nox --sessions lint + - name: Tests env: POSTGRES_HOST: localhost From 90266a483a48c4d1b49f2189807013293cfd7946 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Wed, 26 Oct 2022 15:53:29 -0300 Subject: [PATCH 38/39] fix --- .pre-commit-config.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4c31056..232cb98 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,16 +13,15 @@ repos: # args: [--max-line-length=88] # exclude: templates/* notebook/* - repo: https://github.com/timothycrosley/isort - rev: 4.3.21 + rev: 5.10.1 hooks: - id: isort name: isort entry: isort - language: system types: [python] exclude: templates/.+|notebook/.+ - repo: https://github.com/psf/black - rev: 19.10b0 + rev: 22.10.0 hooks: - id: black name: black @@ -32,7 +31,7 @@ repos: args: [--skip-string-normalization] exclude: templates/.+|notebook/.+ - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.761 + rev: v0.982 hooks: - id: mypy name: mypy @@ -42,7 +41,7 @@ repos: files: \.py$ exclude: templates/.+|notebook/.+ - repo: https://github.com/pre-commit/mirrors-pylint - rev: v2.4.4 + rev: v3.0.0a5 hooks: - id: pylint name: pylint @@ -60,7 +59,7 @@ repos: types: [file] files: 'README.md' require_serial: true - + # exclude: templates/.+|notebook/.+ # repos: # - repo: https://github.com/pre-commit/pre-commit-hooks.git From a553bbfcb44c723568b4d23a66ccc8e789015a39 Mon Sep 17 00:00:00 2001 From: Mauro E Bordon Date: Wed, 26 Oct 2022 15:55:10 -0300 Subject: [PATCH 39/39] text --- .github/workflows/dev.workflow.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dev.workflow.yml b/.github/workflows/dev.workflow.yml index 81f25ae..a1470ec 100644 --- a/.github/workflows/dev.workflow.yml +++ b/.github/workflows/dev.workflow.yml @@ -14,7 +14,7 @@ env: jobs: validate: - runs-on: [self-hosted] + runs-on: ubuntu-latest container: python:3.8.5 env: TARGET_BRANCH: ${{ github.base_ref }} @@ -36,8 +36,9 @@ jobs: added_lines=$(git diff --numstat origin/$TARGET_BRANCH origin/$HEAD_BRANCH -- CHANGELOG.md | awk '{print $1}') if [ -z $added_lines ] || [ $added_lines -eq 0 ]; then echo "Changelog has not been modified" && exit 1; else echo "Changelog has been modified" && exit 0; fi; + test_stage: - runs-on: [self-hosted] + runs-on: ubuntu-latest needs: validate container: image: python:3.8.5 @@ -49,7 +50,7 @@ jobs: POSTGRES_DB: soam_db POSTGRES_USER: soam_usr POSTGRES_PASSWORD: soam_pass - + options: >- --health-cmd pg_isready --health-interval 10s @@ -96,7 +97,7 @@ jobs: nox --sessions pyreverse # version_stage: - # runs-on: [self-hosted] + # runs-on: ubuntu-latest # container: python:3.8.5 # steps: @@ -113,7 +114,7 @@ jobs: release_stage: name: Release Stage - runs-on: [self-hosted] + runs-on: ubuntu-latest needs: test_stage steps: - name: Checkout code