DB v4 progress

.github/actions/setup_rust/action.yml

@@ -15,3 +15,4 @@ runs:
       uses: dtolnay/rust-toolchain@master
       with:
         toolchain: ${{ steps.rust-version.outputs.version }}
+        components: cargo,clippy,rust-docs,rust-src,rust-std,rustc,rustfmt

.github/workflows/_release_docker.yml

@@ -38,9 +38,10 @@ jobs:
       - name: Print input
         run: echo '${{ toJSON(inputs) }}'
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           ref: ${{ inputs.base }}
+          token: '${{ secrets.PULL_TOKEN }}'
 
       - name: Read rust version
         run: echo "RUST_VERSION=$(cargo metadata --no-deps --format-version 1 | jq -r '.packages[0].rust_version')" >> $GITHUB_ENV

.github/workflows/_release_github.yml

@@ -24,7 +24,10 @@ jobs:
         include:
           - os: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
+        with:
+          submodules: recursive
+          token: '${{ secrets.PULL_TOKEN }}'
       - run: |
           git config --global user.email "ben.steer@pometry.com"
           git config --global user.name "Pometry-Team"

.github/workflows/_release_python.yml

@@ -38,8 +38,10 @@ jobs:
             ~/.cargo/git/db/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: ${{ runner.os }}-cargo-
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
+          submodules: recursive
+          token: '${{ secrets.PULL_TOKEN }}'
           ref: ${{ inputs.base }}
       - uses: actions/setup-python@v4
         with:
@@ -130,9 +132,11 @@ jobs:
         uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           ref: ${{ inputs.base }}
+          submodules: recursive
+          token: '${{ secrets.PULL_TOKEN }}'
       - uses: actions/setup-python@v4
         with:
           python-version: |
@@ -174,9 +178,11 @@ jobs:
             ~/.cargo/git/db/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: ${{ runner.os }}-cargo-
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         with:
           ref: ${{ inputs.base }}
+          submodules: recursive
+          token: '${{ secrets.PULL_TOKEN }}'
       - uses: actions/setup-python@v4
         with:
           python-version: |
@@ -219,7 +225,7 @@ jobs:
           skip_existing: true
           verify_metadata: false
       - name: Deleting artifacts
-        uses: geekyeggo/delete-artifact@v2
+        uses: geekyeggo/delete-artifact@v6
         with:
           failOnError: false
           name: |

.github/workflows/_release_rust.yml

@@ -35,8 +35,10 @@ jobs:
         uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
+          submodules: recursive
+          token: '${{ secrets.PULL_TOKEN }}'
           ref: ${{ inputs.base }}
       - name: Setup rust
         uses: dtolnay/rust-toolchain@master

.github/workflows/bench-graphql.yml

@@ -14,8 +14,11 @@ jobs:
     name: GraphQL Benchmark
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         name: Checkout
+        with:
+          submodules: recursive
+          token: '${{ secrets.PULL_TOKEN }}'
       - name: Install Protoc
         uses: arduino/setup-protoc@v3
         with:
@@ -29,7 +32,7 @@ jobs:
       - name: Build raphtory
         run: make install-python
       - name: Set up pnpm
-        uses: pnpm/action-setup@v4
+        uses: pnpm/action-setup@v5
         with:
           version: 9
       - uses: grafana/setup-k6-action@v1
@@ -38,7 +41,7 @@ jobs:
       - name: Run GraphQL benchmarks
         run: cd graphql-bench && make bench-local
       - name: Restore metadata file
-        run: git restore graphql-bench/data/apache/master/.raph # otherwise github-action-benchmark fails to create the commit
+        run: git restore graphql-bench/data/apache/master # otherwise github-action-benchmark fails to create the commit
       - name: Print bench results
         run: cat graphql-bench/output.json
       - name: Store benchmark results from master branch

.github/workflows/benchmark.yml

@@ -26,8 +26,11 @@ jobs:
         include:
           - os: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         name: Checkout
+        with:
+          submodules: recursive
+          token: '${{ secrets.PULL_TOKEN }}'
       - uses: ./.github/actions/setup_rust
         name: Setup Rust
       - name: Install Protoc
@@ -38,7 +41,7 @@ jobs:
         uses: Swatinem/rust-cache@v2
         with:
           cache-all-crates: true
-      - uses: webfactory/ssh-agent@v0.9.0
+      - uses: webfactory/ssh-agent@v0.10.0
         name: Load raphtory-disk_graph key
         with:
           ssh-private-key: ${{ secrets.RA_SSH_PRIVATE_KEY }}

.github/workflows/code_coverage.yml

@@ -18,8 +18,12 @@
        SCCACHE_DIR: /home/runner/.cache/sccache
        SCCACHE_PATH: /home/runner/.cache/sccache
      steps:
-       - uses: actions/checkout@v3
+       - uses: actions/checkout@v5
          name: Checkout
+         with:
+           ref: ${{ github.head_ref }}
+           submodules: recursive
+           token: '${{ secrets.PULL_TOKEN }}'
        - name: Cache sccache
          uses: actions/cache@v3
          with:

.github/workflows/format.yml

@@ -8,10 +8,13 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         name: Checkout
         with:
           ref: ${{ github.head_ref }}
+          submodules: recursive
+          token: '${{ secrets.PULL_TOKEN }}'
+          persist-credentials: 'false'
       - uses: ./.github/actions/setup_rust
         name: Setup Rust
       - name: Setup nightly rust
@@ -34,7 +37,7 @@ jobs:
         run: |
           python -m pip install black mypy pandas-stubs
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v6
         with:
           node-version: '20'
       - name: Install raphtory
@@ -76,10 +79,9 @@ jobs:
             echo "Changes detected. Committing and pushing..."
             git config user.name "github-actions[bot]"
             git config user.email "github-actions[bot]@users.noreply.github.com"
-            git checkout ${{ github.head_ref }}
             git add .
             git commit -m "chore: apply tidy-public auto-fixes"
-            git push --force-with-lease origin HEAD:${{ github.head_ref }} 
+            git push https://oauth2:$GITHUB_TOKEN@github.com/Pometry/Raphtory.git HEAD:${{ github.head_ref }} 
           else
             echo "No changes to commit."
           fi

.github/workflows/manual_release_docker_cloud.yml

@@ -19,8 +19,10 @@ jobs:
   docker:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
+          submodules: recursive
+          token: '${{ secrets.PULL_TOKEN }}'
           ref: ${{ inputs.base }}
 
       - name: Read rust version

.github/workflows/nightly_release.yml

@@ -15,7 +15,10 @@ jobs:
     outputs:
       should_run: ${{ steps.should_run.outputs.should_run }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
+        with:
+          submodules: recursive
+          token: '${{ secrets.PULL_TOKEN }}'
       - id: should_run
         continue-on-error: true
         name: Check latest commit is less than a day

.github/workflows/release_auto.yml

@@ -22,9 +22,10 @@ jobs:
     outputs:
       version: ${{ steps.version.outputs.version }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           ref: ${{ inputs.base }}
+          token: '${{ secrets.PULL_TOKEN }}'
       - id: version
         run: echo "version=$(make print-version)" >> $GITHUB_OUTPUT
   call-release-rust-workflow:

.github/workflows/release_bump_versions.yml

@@ -46,7 +46,10 @@ jobs:
             ~/.cargo/git/db/
           key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
           restore-keys: ${{ runner.os }}-cargo-
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
+        with:
+          submodules: recursive
+          token: '${{ secrets.PULL_TOKEN }}'
       - run: |
           git config --global user.email "ben.steer@pometry.com"
           git config --global user.name "Pometry-Team"
@@ -70,7 +73,7 @@ jobs:
           echo CRATE_VERSION=$(cat Cargo.toml | grep version | head -n1 | cut -d '"' -f2) >> $GITHUB_ENV
           echo "Crate version is $CRATE_VERSION"
       - name: "Make a PR to bump version"
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v8
         with:
           base: ${{ inputs.base }}
           author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>

.github/workflows/rust_format_check.yml

@@ -15,7 +15,10 @@ jobs:
   rust-format-check:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
+        with:
+          submodules: recursive
+          token: '${{ secrets.PULL_TOKEN }}'
       - name: Setup nightly rust
         run: |
           rustup toolchain install nightly --allow-downgrade -c rustfmt

.github/workflows/stress-test.yml

@@ -11,8 +11,11 @@ jobs:
     name: GraphQL Stress Test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v5
         name: Checkout
+        with:
+          submodules: recursive
+          token: '${{ secrets.PULL_TOKEN }}'
       - name: Install Protoc
         uses: arduino/setup-protoc@v3
         with:
@@ -21,21 +24,22 @@ jobs:
         uses: Swatinem/rust-cache@v2
         with:
           cache-all-crates: true
-      - name: Install maturin
-        run: pip install maturin==1.8.3
-      - name: Build raphtory
-        run: make install-python
+      - uses: ./.github/actions/setup_rust
+        name: Setup Rust
       - name: Set up pnpm
-        uses: pnpm/action-setup@v4
+        uses: pnpm/action-setup@v5
         with:
           version: 9
       - uses: grafana/setup-k6-action@v1
         with:
           k6-version: '1.0.0'
       - name: Run stress test
+        env:
+          RUST_BACKTRACE: 1
         run: |
+          cargo build --package raphtory-graphql --bin raphtory-graphql --profile=build-fast
+          ./target/build-fast/raphtory-graphql server --work-dir graphs &
           cd graphql-bench
-          raphtory server &
           make stress-test
       - name: Upload k6 report
         if: always()

.github/workflows/test.yml

@@ -17,22 +17,12 @@ jobs:
     name: Run Rust tests
     uses: ./.github/workflows/test_rust_workflow.yml
     secrets: inherit
-  call-test-rust-storage-workflow-in-local-repo:
-    name: Run Rust storage tests
-    uses: ./.github/workflows/test_rust_disk_storage_workflow.yml
-    secrets: inherit
   call-test-python-workflow-in-local-repo:
     name: Run Python tests
     uses: ./.github/workflows/test_python_workflow.yml
     with:
       test_python_lower: true
     secrets: inherit
-  call-test-python-disk-storage-workflow-in-local-repo:
-    name: Run Python storage tests
-    uses: ./.github/workflows/test_python_disk_storage_workflow.yml
-    with:
-      test_python_lower: true
-    secrets: inherit
   call-test-ui:
     name: Run UI Tests
     uses: ./.github/workflows/test_ui.yml

.github/workflows/test_during_pr.yml

@@ -3,7 +3,8 @@ on:
   pull_request:
     branches:
       - master
-      - "0.16"
+      - db_v4
+      - v16
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -14,22 +15,12 @@ jobs:
     name: Run Rust tests
     uses: ./.github/workflows/test_rust_workflow.yml
     secrets: inherit
-  call-test-rust-storage-workflow-in-local-repo:
-    name: Run Rust storage tests
-    uses: ./.github/workflows/test_rust_disk_storage_workflow.yml
-    secrets: inherit
   call-test-python-workflow-in-local-repo:
     name: Run Python tests
     uses: ./.github/workflows/test_python_workflow.yml
     with:
       test_python_lower: false
     secrets: inherit
-  call-test-python-disk-storage-workflow-in-local-repo:
-    name: Run Python storage tests
-    uses: ./.github/workflows/test_python_disk_storage_workflow.yml
-    with:
-      test_python_lower: false
-    secrets: inherit
   call-test-ui-in-local-repo:
     name: Run UI Tests
     uses: ./.github/workflows/test_ui.yml
@@ -54,10 +45,8 @@ jobs:
     secrets: inherit
     needs: [
       call-test-rust-workflow-in-local-repo,
-      call-test-rust-storage-workflow-in-local-repo,
       call-test-python-workflow-in-local-repo,
-      call-test-python-disk-storage-workflow-in-local-repo,
-      call-test-ui-in-local-repo,
+      #      call-test-ui-in-local-repo,
       call-benchmark-workflow-in-local-repo,
       call-graphql-bench-workflow-in-local-repo,
       call-stress-test-workflow-in-local-repo