From 7889ed6b7ea40b8e9f49820744ad471d32f7c33b Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Thu, 24 Apr 2025 09:05:24 -0400 Subject: [PATCH 01/19] Added caching to npm test job based on packeg.json --- .github/workflows/tests.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7e36e3c05..9204b8910 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -53,4 +53,24 @@ jobs: run: docker compose run page-turner /htapps/babel/pt/bin/build.sh - name: Run playwright tests (PageTurner) - run: docker compose run playwright + # run: docker compose run playwright + + uses: actions/cache@v3 + id: playwright-cache + with: + path: | + ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }} + + - name: Install Dependencies + run: npm ci + + - name: Install Playwright Browsers + run: npx playwright install --with-deps + if: steps.playwright-cache.outputs.cache-hit != 'true' + + - name: Install Playwright Browsers - Cached + run: npx playwright install-deps + if: steps.playwright-cache.outputs.cache-hit == 'true' + - name: Run tests + run: npm test From d58c3d5deab65d941dc0705b6aed599b7400aa7c Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Thu, 24 Apr 2025 10:09:12 -0400 Subject: [PATCH 02/19] Added package.json location --- .github/workflows/tests.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9204b8910..d0bfa22f3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -8,26 +8,26 @@ on: pull_request: jobs: - perl_test: - runs-on: ubuntu-latest + # perl_test: + # runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 + # steps: + # - uses: actions/checkout@v4 - - uses: actions/checkout@v4 - with: - repository: hathitrust/imgsrv-sample-data - path: sample-data + # - uses: actions/checkout@v4 + # with: + # repository: hathitrust/imgsrv-sample-data + # path: sample-data - - name: setup env - run: /bin/bash ./setup-env.sh + # - name: setup env + # run: /bin/bash ./setup-env.sh - - name: Run perl tests - run: docker compose run perl-test + # - name: Run perl tests + # run: docker compose run perl-test playwright: runs-on: ubuntu-latest - needs: perl_test + # needs: perl_test steps: # Check out this repo by default @@ -63,7 +63,7 @@ jobs: key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }} - name: Install Dependencies - run: npm ci + run: npm ci /firebird-common/ - name: Install Playwright Browsers run: npx playwright install --with-deps From a4f6f91f1092a55aca08035b37a570d7e2306569 Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Thu, 24 Apr 2025 10:45:48 -0400 Subject: [PATCH 03/19] Added ls to view tree --- .github/workflows/tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d0bfa22f3..ce9dfd08b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,6 +43,9 @@ jobs: repository: hathitrust/firebird-common path: firebird-common + - name: See env + run: ls -lh + - name: setup env run: /bin/bash ./setup-env.sh From e46d7d6233798bfea6f7913f87c39e0cd2fc8178 Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Thu, 24 Apr 2025 10:50:08 -0400 Subject: [PATCH 04/19] Creating hash from /firebird-common/package.json --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ce9dfd08b..81c14f773 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -63,10 +63,10 @@ jobs: with: path: | ~/.cache/ms-playwright - key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }} + key: ${{ runner.os }}-playwright-${{ hashFiles('firebird-common/package-lock.json') }} - name: Install Dependencies - run: npm ci /firebird-common/ + run: npm ci /firebird-common/package.json - name: Install Playwright Browsers run: npx playwright install --with-deps From 61eaa6cbf694237bde9d85df7325dcd213bcb054 Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Thu, 24 Apr 2025 11:02:20 -0400 Subject: [PATCH 05/19] Added a cd --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 81c14f773..8ab092f78 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -66,7 +66,7 @@ jobs: key: ${{ runner.os }}-playwright-${{ hashFiles('firebird-common/package-lock.json') }} - name: Install Dependencies - run: npm ci /firebird-common/package.json + run: cd firebird-common && npm ci - name: Install Playwright Browsers run: npx playwright install --with-deps From 780149f613dd8d8a92129bf08b8aac0f79cb725e Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Thu, 24 Apr 2025 11:09:04 -0400 Subject: [PATCH 06/19] Working with wrong firebird location --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8ab092f78..8dc9fa620 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -63,10 +63,10 @@ jobs: with: path: | ~/.cache/ms-playwright - key: ${{ runner.os }}-playwright-${{ hashFiles('firebird-common/package-lock.json') }} + key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }} - name: Install Dependencies - run: cd firebird-common && npm ci + run: cd pt/web/firebird/ && npm ci - name: Install Playwright Browsers run: npx playwright install --with-deps From 555031e08eb90c2d407e4ae5114f189d97236f44 Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Thu, 24 Apr 2025 11:23:34 -0400 Subject: [PATCH 07/19] Commenting out `npm ci` for a test --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8dc9fa620..29d6dc314 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -65,8 +65,8 @@ jobs: ~/.cache/ms-playwright key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }} - - name: Install Dependencies - run: cd pt/web/firebird/ && npm ci + # - name: Install Dependencies + # run: npm ci - name: Install Playwright Browsers run: npx playwright install --with-deps From ef21bcc26a5c72b308656cb6366fdf880f9c1a55 Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Thu, 24 Apr 2025 11:38:24 -0400 Subject: [PATCH 08/19] added pt directory --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 29d6dc314..3c8ce4bb4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -76,4 +76,4 @@ jobs: run: npx playwright install-deps if: steps.playwright-cache.outputs.cache-hit == 'true' - name: Run tests - run: npm test + run: npm test pt/web/firebird/ From 0cdc872e178c53cb74bf6667475b85162757e68c Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Thu, 24 Apr 2025 11:56:20 -0400 Subject: [PATCH 09/19] Added `--prefi` flag to `package.json` for `npm`. --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3c8ce4bb4..64f8bb2bf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -66,7 +66,7 @@ jobs: key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }} # - name: Install Dependencies - # run: npm ci + # run: npm --prefix pt/web/firebird/ ci - name: Install Playwright Browsers run: npx playwright install --with-deps @@ -76,4 +76,4 @@ jobs: run: npx playwright install-deps if: steps.playwright-cache.outputs.cache-hit == 'true' - name: Run tests - run: npm test pt/web/firebird/ + run: npm --prefix pt/web/firebird/ test From e7b2c4200ae7772713ce298ff72cb89fd51b7a03 Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Thu, 24 Apr 2025 12:25:17 -0400 Subject: [PATCH 10/19] Added line to install dependencies --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 64f8bb2bf..72d0c6042 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -65,8 +65,8 @@ jobs: ~/.cache/ms-playwright key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }} - # - name: Install Dependencies - # run: npm --prefix pt/web/firebird/ ci + - name: Install Dependencies + run: npm --prefix pt/web/firebird/ ci - name: Install Playwright Browsers run: npx playwright install --with-deps From 8a59cf9195e745c1c411cadbe024a77d00d519d1 Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Thu, 24 Apr 2025 14:20:27 -0400 Subject: [PATCH 11/19] Adding node to gh runner. --- .github/workflows/tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 72d0c6042..b9bb11cee 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -66,7 +66,10 @@ jobs: key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }} - name: Install Dependencies - run: npm --prefix pt/web/firebird/ ci + uses: actions/setup-node@v4 + with: + node-version: 18 + - run: npm --prefix pt/web/firebird/ ci - name: Install Playwright Browsers run: npx playwright install --with-deps From 23c5d30eee0a7091f2e6e8d23668a8129e575e6a Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Mon, 28 Apr 2025 11:36:44 -0400 Subject: [PATCH 12/19] Updated prefix to firebird-common --- .github/workflows/tests.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b9bb11cee..e7d6b842e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -69,14 +69,15 @@ jobs: uses: actions/setup-node@v4 with: node-version: 18 - - run: npm --prefix pt/web/firebird/ ci + - run: npm --prefix firebird-common/ ci - name: Install Playwright Browsers - run: npx playwright install --with-deps if: steps.playwright-cache.outputs.cache-hit != 'true' + run: npx playwright install --with-deps - name: Install Playwright Browsers - Cached - run: npx playwright install-deps if: steps.playwright-cache.outputs.cache-hit == 'true' + run: npx playwright install-deps + - name: Run tests - run: npm --prefix pt/web/firebird/ test + run: npm --prefix firebird-common/ test From 01f7837e1b5f11dff3b9c97c69f92db520d6621a Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Mon, 28 Apr 2025 12:11:25 -0400 Subject: [PATCH 13/19] Added caching for Docker Compose instead of npm. --- .github/workflows/tests.yml | 35 ++++++----------------------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e09c37996..748f83e08 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,45 +43,22 @@ jobs: repository: hathitrust/firebird-common path: firebird-common - - name: See env - run: ls -lh - - name: setup env run: /bin/bash ./setup-env.sh + - name: Setup Docker Compose + uses: docker/setup-compose-action@v1.2.0 + with: + version: latest + cache-binary: true + - name: Build firebird run: docker compose run firebird ./bin/build.sh - name: Build pageturner run: docker compose run page-turner /htapps/babel/pt/bin/build.sh - - name: Run playwright tests (PageTurner) - # run: docker compose run playwright - - uses: actions/cache@v3 - id: playwright-cache - with: - path: | - ~/.cache/ms-playwright - key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }} - - - name: Install Dependencies - uses: actions/setup-node@v4 - with: - node-version: 18 - - run: npm --prefix firebird-common/ ci - - - name: Install Playwright Browsers - if: steps.playwright-cache.outputs.cache-hit != 'true' - run: npx playwright install --with-deps - - - name: Install Playwright Browsers - Cached - if: steps.playwright-cache.outputs.cache-hit == 'true' - run: npx playwright install-deps - - name: Run tests - # run: npm --prefix firebird-common/ test - run: docker compose run playwright # - uses: actions/upload-artifact@v4 From 763f71bcfc37ae4ec55d6021280ab535c40616a8 Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Tue, 29 Apr 2025 10:03:09 -0400 Subject: [PATCH 14/19] Added caching for each service being built here. --- .github/workflows/tests.yml | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 748f83e08..67b064194 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,14 +43,25 @@ jobs: repository: hathitrust/firebird-common path: firebird-common + - name: See env + run: ls -lh + - name: setup env run: /bin/bash ./setup-env.sh - - name: Setup Docker Compose - uses: docker/setup-compose-action@v1.2.0 + - name: Build and Check Cache + uses: docker/bake-action@master with: - version: latest - cache-binary: true + push: false + load: true # Load from cache + set: | + firebird.cache-from=type=gha + firebird.cache-to=type=gha + page-turner.cache-from=type=gha + page-turner.cache-to=type=gha + playwright.cache-from=type=gha + playwright.cache-to=type=gha + - name: Build firebird run: docker compose run firebird ./bin/build.sh @@ -58,12 +69,12 @@ jobs: - name: Build pageturner run: docker compose run page-turner /htapps/babel/pt/bin/build.sh - - name: Run tests + - name: Run playwright tests (PageTurner) run: docker compose run playwright -# - uses: actions/upload-artifact@v4 -# if: ${{ !cancelled() }} -# with: -# name: test-results -# path: pt/web/firebird/test-results/ -# retention-days: 30 + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: test-results + path: pt/web/firebird/test-results/ + retention-days: 30 \ No newline at end of file From a160b44f381f9b1569cba1dfbd21e8faab0d92e6 Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Tue, 29 Apr 2025 10:24:16 -0400 Subject: [PATCH 15/19] Falling back to the cache action for layers --- .github/workflows/tests.yml | 41 +++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 67b064194..87efbc4da 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,19 +49,26 @@ jobs: - name: setup env run: /bin/bash ./setup-env.sh - - name: Build and Check Cache - uses: docker/bake-action@master + # - name: Build and Check Cache + # uses: docker/bake-action@master + # with: + # push: false + # load: true # Load from cache + # set: | + # firebird.cache-from=type=gha + # firebird.cache-to=type=gha + # page-turner.cache-from=type=gha + # page-turner.cache-to=type=gha + # playwright.cache-from=type=gha + # playwright.cache-to=type=gha + + - name: Cache Docker layers + uses: actions/cache@v4 with: - push: false - load: true # Load from cache - set: | - firebird.cache-from=type=gha - firebird.cache-to=type=gha - page-turner.cache-from=type=gha - page-turner.cache-to=type=gha - playwright.cache-from=type=gha - playwright.cache-to=type=gha - + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Build firebird run: docker compose run firebird ./bin/build.sh @@ -77,4 +84,12 @@ jobs: with: name: test-results path: pt/web/firebird/test-results/ - retention-days: 30 \ No newline at end of file + retention-days: 30\ + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache \ No newline at end of file From f8c3fdc1424791691a890ab4a65da0cbc1415465 Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Tue, 29 Apr 2025 10:34:34 -0400 Subject: [PATCH 16/19] Removed temporary cache move for now. --- .github/workflows/tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 87efbc4da..95684094f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -89,7 +89,7 @@ jobs: # Temp fix # https://github.com/docker/build-push-action/issues/252 # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache \ No newline at end of file + # - name: Move cache + # run: | + # rm -rf /tmp/.buildx-cache + # mv /tmp/.buildx-cache-new /tmp/.buildx-cache \ No newline at end of file From cc3f19f13014df1522b311ec6f86d4eefc04bcd7 Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Fri, 2 May 2025 12:22:48 -0400 Subject: [PATCH 17/19] Added cache set up --- .github/workflows/tests.yml | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 95684094f..25dcdd7a4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -43,9 +43,6 @@ jobs: repository: hathitrust/firebird-common path: firebird-common - - name: See env - run: ls -lh - - name: setup env run: /bin/bash ./setup-env.sh @@ -62,13 +59,18 @@ jobs: # playwright.cache-from=type=gha # playwright.cache-to=type=gha + - name: Set Cache Location + run: touch ${{ runner.temp }}/.buildx-cache + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Cache Docker layers uses: actions/cache@v4 with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- + path: ${{ runner.temp }}/.buildx-cache + key: ${{ runner.os }}-buildx-${{ hashFiles('**/package.json') }} + restore-keys: ${{ runner.os }}-buildx- - name: Build firebird run: docker compose run firebird ./bin/build.sh @@ -84,12 +86,4 @@ jobs: with: name: test-results path: pt/web/firebird/test-results/ - retention-days: 30\ - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - # - name: Move cache - # run: | - # rm -rf /tmp/.buildx-cache - # mv /tmp/.buildx-cache-new /tmp/.buildx-cache \ No newline at end of file + retention-days: 30 From 48a90e178e076d3ca67c03fc6ca0f86a520dede3 Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Fri, 2 May 2025 12:35:24 -0400 Subject: [PATCH 18/19] Minor change to trigger another run. --- .github/workflows/tests.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 25dcdd7a4..f6f11cd5e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -46,19 +46,6 @@ jobs: - name: setup env run: /bin/bash ./setup-env.sh - # - name: Build and Check Cache - # uses: docker/bake-action@master - # with: - # push: false - # load: true # Load from cache - # set: | - # firebird.cache-from=type=gha - # firebird.cache-to=type=gha - # page-turner.cache-from=type=gha - # page-turner.cache-to=type=gha - # playwright.cache-from=type=gha - # playwright.cache-to=type=gha - - name: Set Cache Location run: touch ${{ runner.temp }}/.buildx-cache From 6b2780b4bd2e52a1a69c67b6a91c2fe0e54fb4cc Mon Sep 17 00:00:00 2001 From: K'Ron Spar Date: Thu, 8 May 2025 09:19:30 -0400 Subject: [PATCH 19/19] Caching `package-lock.json` instead of just the `package.json`. --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f6f11cd5e..bc7865ed9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -56,7 +56,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ runner.temp }}/.buildx-cache - key: ${{ runner.os }}-buildx-${{ hashFiles('**/package.json') }} + key: ${{ runner.os }}-buildx-${{ hashFiles('**/package-lock.json') }} restore-keys: ${{ runner.os }}-buildx- - name: Build firebird