diff --git a/CODEOWNERS b/CODEOWNERS index 94c5e1b7..6dda68b4 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -41,4 +41,5 @@ /tools/sf-dependency-list @yuriylesyuk /tools/target-server-validator @anaik91 /tools/apigee-proxy-modifier-validator @anaik91 -/tools/apigee-edge-to-x-migration-tool @h-kopalle \ No newline at end of file +/tools/apigee-edge-to-x-migration-tool @h-kopalle +/tools/apigee-config-diff @rickygodoy diff --git a/README.md b/README.md index dc88363b..277f5a37 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,8 @@ Apigee products. - [Apigee Proxy Bundle Modifier & Validator](tools/apigee-proxy-modifier-validator) - A tool to do batch modifications of API proxies when migrating to newer Apigee variants. - [Apigee Edge to X Migration Accelerator](/tools/apigee-edge-to-x-migration-tool) - A tool built on node.js to accelerate Apigee Edge to X migration. +- [Apigee Config Diff Tool](tools/apigee-config-diff) - + A tool to find and deploy only the changes in Apigee configurations. ## Labs diff --git a/tools/apigee-config-diff/.bandit b/tools/apigee-config-diff/.bandit new file mode 100644 index 00000000..7438283b --- /dev/null +++ b/tools/apigee-config-diff/.bandit @@ -0,0 +1,2 @@ +[bandit] +exclude = tests diff --git a/tools/apigee-config-diff/.gitignore b/tools/apigee-config-diff/.gitignore new file mode 100644 index 00000000..50d7ee34 --- /dev/null +++ b/tools/apigee-config-diff/.gitignore @@ -0,0 +1,12 @@ +.DS_Store +.idea +*.log +tmp/ + +*.py[cod] +*.egg +build +htmlcov +venv +output +__pycache__ diff --git a/tools/apigee-config-diff/README.md b/tools/apigee-config-diff/README.md new file mode 100644 index 00000000..927a6e02 --- /dev/null +++ b/tools/apigee-config-diff/README.md @@ -0,0 +1,115 @@ +

+ Apigee Config Diff Generator Logo +

+ +# Apigee Config Diff Tool + +**Deploy only what changed in your Apigee configurations.** + +This tool finds differences in Apigee Configuration files (KVMs, Target Servers, API Products, etc.) between two Git commits and uses the [Apigee Config Maven Plugin](https://github.com/apigee/apigee-config-maven-plugin) to apply only those changes. + +## Why use this? + +Standard Apigee configuration deployments often re-deploy the entire configuration set, which can be slow and risky for large organizations. This tool allows for **incremental deployments**: +- **Speed:** Only process files and resources that actually changed. +- **Safety:** Reduce the risk of accidental overwrites of stable configurations. +- **CI/CD Friendly:** Designed to run in pipelines (Jenkins, GitHub Actions, GitLab, etc.). + +--- + +## How it Works + +1. **Diff:** Compares two Git commits (e.g., `HEAD~1` vs `HEAD`). +2. **Generate:** Creates two temporary trees in `output/`: + - `update/`: New or modified resources. + - `delete/`: Resources removed or items deleted from within modified files. +3. **Deploy:** Runs `mvn install` on both trees with the `update` or `delete` option. + +--- + +## Quick Start + +### 1. Requirements +- Python 3.10+ +- Maven (`mvn`) installed and in your PATH. +- (Optional) `tree` command for better dry-run visualization. + +### 2. Installation +Install directly from GitHub: +```bash +pip install git+https://github.com/apigee/devrel.git#subdirectory=tools/apigee-config-diff +``` + +### 3. Basic Usage (Dry Run) +By default, the tool compares `HEAD~1` with `HEAD` in the `resources/` directory. +```bash +apigee-config-diff +``` + +### 4. Deploy Changes +Add the `--confirm` flag to actually execute the Maven commands. +```bash +apigee-config-diff --confirm +``` + +--- + +## Configuration & Arguments + +| Argument | Default | Description | +| :--- | :--- | :--- | +| `--commit-before` | `HEAD~1` | Previous commit hash to compare. | +| `--current-commit` | `HEAD` | Current commit hash. | +| `--folder` | `resources` | Folder containing the Apigee config tree. | +| `--output` | `output` | Where to generate the temporary diff trees. | +| `--confirm` | `False` | Must be present to execute `mvn` commands. | +| `--bearer` | `None` | Optional: Apigee bearer token to use for Maven. | +| `--sa-path` | `None` | Optional: Path to the service account key file. | + +### Authentication +The tool passes authentication flags directly to the Maven command. You can provide them in three ways: + +1. **Command Line (Recommended):** + ```bash + apigee-config-diff --confirm --bearer "$(gcloud auth print-access-token)" + # OR + apigee-config-diff --confirm --sa-path /tmp/sa.json + ``` + +2. **Environment Variables:** + If your `pom.xml` is configured to use environment variables (e.g., `${env.bearer}`), simply export them before running the script: + ```bash + export bearer=$(gcloud auth print-access-token) + apigee-config-diff --confirm + ``` + +3. **POM Configuration:** + Hardcode the service account path or token directly in your `pom.xml` (not recommended for CI/CD). + +--- + +## Expected Folder Structure +The tool expects the standard [Maven config structure](https://github.com/apigee/apigee-config-maven-plugin?tab=readme-ov-file#multi-file-config) inside your `--folder` (default `resources/`): + +```text +/ + ├── org/ + │ ├── apiProducts.json + │ ├── developers.json + │ └── ... + └── env/ + ├── / + │ ├── targetServers.json + │ ├── kvms.json + │ └── ... +``` + +***Note**: File names must start with the resource type (e.g., `targetServers-backend.json` is valid).* + +--- + +## Contributing +Run tests with: +```bash +pytest +``` diff --git a/tools/apigee-config-diff/example/pom.xml b/tools/apigee-config-diff/example/pom.xml new file mode 100644 index 00000000..4d0b12bc --- /dev/null +++ b/tools/apigee-config-diff/example/pom.xml @@ -0,0 +1,200 @@ + + + + 4.0.0 + apigee + parent-pom + pom + 1.0 + + + central + Maven Plugin Repository + https://repo1.maven.org/maven2 + default + + false + + + never + + + + + + + + com.apigee.edge.config + apigee-config-maven-plugin + 2.9.3 + + + create-config-targetserver + install + + targetservers + + + + create-config-kvm + install + + keyvaluemaps + + + + create-config-keystores + install + + keystores + + + + create-config-aliases + install + + aliases + + + + create-config-references + install + + references + + + + create-config-resourcefiles + install + + resourcefiles + + + + create-config-apiproduct + install + + apiproducts + + + + create-config-rateplans + install + + rateplans + + + + create-config-developer + install + + developers + + + + create-config-app + install + + apps + + + + create-config-appgroups + install + + appgroups + + + + create-config-appgroupapps + install + + appgroupapps + + + + create-config-reports + install + + reports + + + + create-config-flowhooks + install + + flowhooks + + + + import-app-keys + install + + importKeys + + + + create-portal-categories + install + + apicategories + + + + create-portal-docs + install + + apidocs + + + + create-spaces + install + + spaces + + + + + + + + + my-org + + my-org + https://apigee.googleapis.com + v1 + ${org} + ${env} + oauth + ${bearer} + ${siteId} + ${file} + + + + + + junit + junit + 4.8.2 + test + + + diff --git a/tools/apigee-config-diff/example/resources/my-org/env/prod/aliases.json b/tools/apigee-config-diff/example/resources/my-org/env/prod/aliases.json new file mode 100644 index 00000000..a6436fd8 --- /dev/null +++ b/tools/apigee-config-diff/example/resources/my-org/env/prod/aliases.json @@ -0,0 +1,13 @@ +[ + { + "alias": "my-alias", + "keystorename": "my-keystore", + "format": "selfsignedcert", + "keySize": "2048", + "sigAlg": "SHA256withRSA", + "subject": { + "commonName": "test.example.com" + }, + "certValidityInDays": "365" + } +] diff --git a/tools/apigee-config-diff/example/resources/my-org/env/prod/cache.json b/tools/apigee-config-diff/example/resources/my-org/env/prod/cache.json new file mode 100644 index 00000000..e4ba455f --- /dev/null +++ b/tools/apigee-config-diff/example/resources/my-org/env/prod/cache.json @@ -0,0 +1,13 @@ +[ + { + "name": "my-cache", + "description": "General purpose cache", + "expirySettings": { + "values": { + "expiryDate": { + "value": "12-31-2025" + } + } + } + } +] diff --git a/tools/apigee-config-diff/example/resources/my-org/env/prod/flowhooks.json b/tools/apigee-config-diff/example/resources/my-org/env/prod/flowhooks.json new file mode 100644 index 00000000..8960833d --- /dev/null +++ b/tools/apigee-config-diff/example/resources/my-org/env/prod/flowhooks.json @@ -0,0 +1,6 @@ +[ + { + "flowHookPoint": "PreProxyFlowHook", + "sharedFlow": "security-shared-flow" + } +] diff --git a/tools/apigee-config-diff/example/resources/my-org/env/prod/keystores.json b/tools/apigee-config-diff/example/resources/my-org/env/prod/keystores.json new file mode 100644 index 00000000..a963d19c --- /dev/null +++ b/tools/apigee-config-diff/example/resources/my-org/env/prod/keystores.json @@ -0,0 +1,5 @@ +[ + { + "name": "my-keystore" + } +] diff --git a/tools/apigee-config-diff/example/resources/my-org/env/prod/kvm.json b/tools/apigee-config-diff/example/resources/my-org/env/prod/kvm.json new file mode 100644 index 00000000..2e4bd6c0 --- /dev/null +++ b/tools/apigee-config-diff/example/resources/my-org/env/prod/kvm.json @@ -0,0 +1,12 @@ +[ + { + "name": "settings", + "encrypted": "true", + "entry": [ + { + "name": "api_key", + "value": "secret-value" + } + ] + } +] diff --git a/tools/apigee-config-diff/example/resources/my-org/env/prod/references.json b/tools/apigee-config-diff/example/resources/my-org/env/prod/references.json new file mode 100644 index 00000000..ac1018eb --- /dev/null +++ b/tools/apigee-config-diff/example/resources/my-org/env/prod/references.json @@ -0,0 +1,7 @@ +[ + { + "name": "my-reference", + "refers": "my-keystore", + "resourceType": "KeyStore" + } +] diff --git a/tools/apigee-config-diff/example/resources/my-org/env/prod/targetServers.json b/tools/apigee-config-diff/example/resources/my-org/env/prod/targetServers.json new file mode 100644 index 00000000..4db2590a --- /dev/null +++ b/tools/apigee-config-diff/example/resources/my-org/env/prod/targetServers.json @@ -0,0 +1,11 @@ +[ + { + "name": "backend-target", + "host": "backend.example.com", + "port": 443, + "isEnabled": true, + "sSLInfo": { + "enabled": "true" + } + } +] diff --git a/tools/apigee-config-diff/example/resources/my-org/org/apiProducts.json b/tools/apigee-config-diff/example/resources/my-org/org/apiProducts.json new file mode 100644 index 00000000..9342c5f8 --- /dev/null +++ b/tools/apigee-config-diff/example/resources/my-org/org/apiProducts.json @@ -0,0 +1,18 @@ +[ + { + "name": "SampleProduct", + "displayName": "Sample Product Edited", + "approvalType": "auto", + "attributes": [ + { + "name": "access", + "value": "public" + } + ], + "environments": ["prod"], + "proxies": ["hello-world"], + "quota": "10000", + "quotaInterval": "1", + "quotaTimeUnit": "month" + } +] diff --git a/tools/apigee-config-diff/example/resources/my-org/org/appGroup.json b/tools/apigee-config-diff/example/resources/my-org/org/appGroup.json new file mode 100644 index 00000000..ff22875b --- /dev/null +++ b/tools/apigee-config-diff/example/resources/my-org/org/appGroup.json @@ -0,0 +1,8 @@ +[ + { + "name": "SampleAppGroup", + "displayName": "Sample App Group", + "attributes": [], + "status": "active" + } +] diff --git a/tools/apigee-config-diff/example/resources/my-org/org/developerApps.json b/tools/apigee-config-diff/example/resources/my-org/org/developerApps.json new file mode 100644 index 00000000..75b61060 --- /dev/null +++ b/tools/apigee-config-diff/example/resources/my-org/org/developerApps.json @@ -0,0 +1,11 @@ +{ + "dev@example.com": [ + { + "name": "SampleApp", + "apiProducts": [ + "SampleProduct" + ], + "callbackUrl": "https://example.com" + } + ] +} diff --git a/tools/apigee-config-diff/example/resources/my-org/org/developers.json b/tools/apigee-config-diff/example/resources/my-org/org/developers.json new file mode 100644 index 00000000..2384fd8c --- /dev/null +++ b/tools/apigee-config-diff/example/resources/my-org/org/developers.json @@ -0,0 +1,9 @@ +[ + { + "email": "dev@example.com", + "firstName": "Sample", + "lastName": "Developer", + "userName": "sampledev", + "attributes": [] + } +] diff --git a/tools/apigee-config-diff/example/resources/my-org/org/reports.json b/tools/apigee-config-diff/example/resources/my-org/org/reports.json new file mode 100644 index 00000000..f7c9d10d --- /dev/null +++ b/tools/apigee-config-diff/example/resources/my-org/org/reports.json @@ -0,0 +1,15 @@ +[ + { + "name": "TrafficReport", + "displayName": "Traffic Report", + "dimensions": ["proxy"], + "metrics": [ + { + "name": "sum(message_count)", + "function": "sum" + } + ], + "chartType": "line", + "timeUnit": "hour" + } +] diff --git a/tools/apigee-config-diff/logo.svg b/tools/apigee-config-diff/logo.svg new file mode 100644 index 00000000..bb74d7ba --- /dev/null +++ b/tools/apigee-config-diff/logo.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + Apigee + Config Diff + diff --git a/tools/apigee-config-diff/pipeline.sh b/tools/apigee-config-diff/pipeline.sh new file mode 100755 index 00000000..5ba4c39e --- /dev/null +++ b/tools/apigee-config-diff/pipeline.sh @@ -0,0 +1,119 @@ +#!/bin/sh + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +SCRIPTPATH="$( + cd "$(dirname "$0")" || exit >/dev/null 2>&1 + pwd -P +)" + +echo "==> Setting up a dummy git repository to exercise apigee-config-diff..." +TEMP_DIR="$(mktemp -d)" +cd "$TEMP_DIR" + +# Initialize git +git init -b main +git config user.name "Apigee Config Diff Pipeline" +git config user.email "pipeline@example.com" + +# Create first commit (initial state) +mkdir -p resources/org +cat <<'IN' > resources/org/targetServers.json +[ + { + "name": "target-1", + "host": "httpbin.org", + "port": 443 + } +] +IN + +cat <<'IN' > resources/org/kvms.json +[ + { + "name": "my-kvm" + } +] +IN + +git add . +git commit -m "Initial commit with targets and kvm" + +# Create second commit (add, modify, delete) +# 1. Modify existing target server list +cat <<'IN' > resources/org/targetServers.json +[ + { + "name": "target-1", + "host": "httpbin.org", + "port": 80 + }, + { + "name": "target-2", + "host": "mocktarget.apigee.net", + "port": 443 + } +] +IN + +# 2. Delete a file +rm resources/org/kvms.json + +# 3. Add a new file +cat <<'IN' > resources/org/apiProducts.json +[ + { + "name": "test-product" + } +] +IN + +git add . +git commit -m "Second commit modifying configs" + +echo "==> Setting up Python virtual environment..." +VENV_PATH="$TEMP_DIR/venv" +python3 -m venv "$VENV_PATH" +# shellcheck source=/dev/null +. "$VENV_PATH/bin/activate" + +echo "==> Installing apigee-config-diff from $SCRIPTPATH..." +# Fallback to python execution if pip install fails in some restrictive environments +pip install --index-url https://pypi.org/simple/ "$SCRIPTPATH" || true + +echo "==> Running apigee-config-diff tool..." +# Run tool with default (HEAD~1 vs HEAD) for folder 'resources' generating in 'output' +if command -v apigee-config-diff >/dev/null 2>&1; then + apigee-config-diff --folder resources --output "$TEMP_DIR/output" +else + export PYTHONPATH="$SCRIPTPATH/src" + python3 -m apigee_config_diff.main --folder resources --output "$TEMP_DIR/output" +fi + +echo "==> Inspecting output directory (Dry run)..." +if command -v tree >/dev/null 2>&1; then + tree "$TEMP_DIR/output" +else + find "$TEMP_DIR/output" -type f +fi + +echo "==> Cleaning up resources..." +deactivate +rm -rf "$TEMP_DIR" +rm -rf "$SCRIPTPATH/src/apigee_config_diff.egg-info" + +echo "==> Pipeline finished successfully." diff --git a/tools/apigee-config-diff/pyproject.toml b/tools/apigee-config-diff/pyproject.toml new file mode 100644 index 00000000..c853429c --- /dev/null +++ b/tools/apigee-config-diff/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "apigee-config-diff" +version = "0.1.0" +description = "Incremental deployment tool for Apigee Maven Config" +readme = "README.md" +requires-python = ">=3.9" +authors = [ + { name = "Apigee DevRel" } +] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", +] +dependencies = [] + +[project.urls] +"Homepage" = "https://github.com/apigee/devrel/tree/main/tools/apigee-config-diff" + +[project.scripts] +apigee-config-diff = "apigee_config_diff.main:main" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["apigee_config_diff*"] + +[tool.pytest.ini_options] +pythonpath = ["src"] +testpaths = ["tests"] diff --git a/tools/apigee-config-diff/src/apigee_config_diff/__init__.py b/tools/apigee-config-diff/src/apigee_config_diff/__init__.py new file mode 100644 index 00000000..75edc9bb --- /dev/null +++ b/tools/apigee-config-diff/src/apigee_config_diff/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tools/apigee-config-diff/src/apigee_config_diff/diff/__init__.py b/tools/apigee-config-diff/src/apigee_config_diff/diff/__init__.py new file mode 100644 index 00000000..75edc9bb --- /dev/null +++ b/tools/apigee-config-diff/src/apigee_config_diff/diff/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tools/apigee-config-diff/src/apigee_config_diff/diff/check.py b/tools/apigee-config-diff/src/apigee_config_diff/diff/check.py new file mode 100644 index 00000000..6bd58cdb --- /dev/null +++ b/tools/apigee-config-diff/src/apigee_config_diff/diff/check.py @@ -0,0 +1,246 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import json + +from .diff import diff +from .util import ( + GitClient, + write_to_file, + find_resource_type, + create_folder, + merge, +) + +RESOURCES_ID = { + "flowhooks": "flowHookPoint", + "references": "name", + "targetServers": "name", + "keystores": "name", + "aliases": "alias", + "apiProducts": "name", + "developers": "email", + "developerApps": "name", + "kvm": "name", + "cache": "name", + "appGroup": "name", + "caches": "name", + "appGroups": "name", + "reports": "name", +} + + +def detect_changes(previous_commit, current_commit, resources_base_path): + + print("==================================================") + print("Detecting File Changes") + print("==================================================") + effective_previous_commit_msg = ( + previous_commit if previous_commit else "N/A (listing all as new)" + ) + print(f" Previous Commit (Effective): {effective_previous_commit_msg}") + print(f" Current Commit: {current_commit}") + print("") + + added_files = [] + modified_files = [] + deleted_files = [] + + if not previous_commit: + result = GitClient.list_files() + if result.stdout: + for file_path in result.stdout.strip().split("\n"): + if file_path and file_path.startswith(resources_base_path): + added_files.append(file_path) + else: + result = GitClient.diff_hashes(previous_commit, current_commit) + diff_output = result.stdout.strip() if result.stdout else "" + + if diff_output: + for line in diff_output.split("\n"): + if not line: + continue + parts = line.split("\t") + status_char = parts[0][0] + path_old = parts[1] + path_new = parts[2] if len(parts) > 2 else None + + if status_char == "A": + if path_old.startswith(resources_base_path): + added_files.append(path_old) + elif status_char == "M": + if path_old.startswith(resources_base_path): + modified_files.append(path_old) + elif status_char == "D": + if path_old.startswith(resources_base_path): + deleted_files.append(path_old) + elif status_char == "R": + if not path_new: + print( + f"Warning: Rename status '{parts[0]}' " + f"for '{path_old}' " + f"missing new path.", + file=sys.stderr, + ) + continue + if path_old.startswith(resources_base_path): + deleted_files.append(path_old) + if path_new.startswith(resources_base_path): + added_files.append(path_new) + elif status_char == "C": + if not path_new: + print( + f"Warning: Copy status '{parts[0]}' " + f"for '{path_old}' " + f"missing new path.", + file=sys.stderr, + ) + continue + if path_new.startswith(resources_base_path): + added_files.append(path_new) + else: + print( + f"Unknown git status: {parts[0]} for file {path_old}", + file=sys.stderr, + ) + + print("--- Summary of Changes ---") + + print(f"Added files ({len(added_files)}):") + if added_files: + for f_path in added_files: + print(f" {f_path}") + + print(f"Deleted files ({len(deleted_files)}):") + if deleted_files: + for f_path in deleted_files: + print(f" {f_path}") + + print(f"Modified files ({len(modified_files)}):") + if modified_files: + for f_path in modified_files: + print(f" {f_path}") + + return added_files, deleted_files, modified_files + + +def calculate_file_diffs( + added_files, deleted_files, modified_files, previous_commit, current_commit +): + """ + Calculates the diffs and determines what content needs to be written + for updates and deletions. + Returns two dictionaries: files_to_update, files_to_delete containing + file_path to json mapping. + """ + files_to_update = {} + files_to_delete = {} + + for f_path in added_files: + file_contents = GitClient.read_file_contents(current_commit, f_path) + try: + files_to_update[f_path] = json.loads(file_contents) + except json.JSONDecodeError as e: + print( + f"Warning: Failed to parse JSON in added file {f_path}: {e}", + file=sys.stderr, + ) + + for f_path in deleted_files: + file_contents = GitClient.read_file_contents(previous_commit, f_path) + try: + files_to_delete[f_path] = json.loads(file_contents) + except json.JSONDecodeError as e: + print( + f"Warning: Failed to parse JSON in deleted file {f_path}: {e}", + file=sys.stderr, + ) + + for f_path in modified_files: + previous_file_contents = GitClient.read_file_contents( + previous_commit, f_path + ) + current_file_contents = GitClient.read_file_contents( + current_commit, f_path + ) + + file_name = os.path.basename(f_path) + resource_type = find_resource_type(file_name, RESOURCES_ID) + + try: + if resource_type: + diff_elements = diff( + json.loads(previous_file_contents), + json.loads(current_file_contents), + RESOURCES_ID[resource_type], + ) + + print(f"Diff of elements inside {f_path}:") + print(json.dumps(diff_elements, indent=4)) + + added_and_modified = merge( + diff_elements["added"], diff_elements["modified"] + ) + + if added_and_modified: + files_to_update[f_path] = added_and_modified + + if diff_elements["deleted"]: + files_to_delete[f_path + ".delete"] = diff_elements[ + "deleted" + ] + else: + print( + f"Unknown resource type for {f_path}. Deploying full file." + ) + files_to_update[f_path] = json.loads(current_file_contents) + except json.JSONDecodeError as e: + print( + f"Warning: Failed to parse JSON " + f"in modified file {f_path}: {e}", + file=sys.stderr, + ) + + return files_to_update, files_to_delete + + +def write_temporary_files( + added_files, + deleted_files, + modified_files, + previous_commit, + current_commit, + tmp_base_path, +): + """ + Resolves the diffs and writes the resulting temporary files to disk. + """ + files_to_update, files_to_delete = calculate_file_diffs( + added_files, + deleted_files, + modified_files, + previous_commit, + current_commit, + ) + + update_folder = create_folder(f"{tmp_base_path}/update") + delete_folder = create_folder(f"{tmp_base_path}/delete") + + for f_path, contents in files_to_update.items(): + write_to_file(os.path.join(update_folder, f_path), contents) + + for f_path, contents in files_to_delete.items(): + write_to_file(os.path.join(delete_folder, f_path), contents) diff --git a/tools/apigee-config-diff/src/apigee_config_diff/diff/diff.py b/tools/apigee-config-diff/src/apigee_config_diff/diff/diff.py new file mode 100644 index 00000000..a0aaeab6 --- /dev/null +++ b/tools/apigee-config-diff/src/apigee_config_diff/diff/diff.py @@ -0,0 +1,193 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Module for diffing Apigee configurations. +Provides functions to recursively diff dictionaries and lists of objects. +""" + +from typing import Any, Dict, List, Union, TypedDict + +JSONValue = Union[ + Dict[str, Any], List[Any], str, int, float, bool, None +] + + +class DiffResult(TypedDict): + """Type definition for the diff result.""" + + added: JSONValue + deleted: JSONValue + modified: JSONValue + + +IdentificationMap = Dict[Any, Dict[str, Any]] + + +def _transform_identification( + identifier: str, content: List[Dict[str, Any]] +) -> "IdentificationMap": + """ + Transforms a list of dictionaries into a dictionary indexed + by the given identifier. + + Args: + identifier: The key to use as the index. + content: The list of dictionaries to transform. + + Returns: + A dictionary where keys are the values of the identifier in each item. + + Raises: + ValueError: If the identifier is empty. + KeyError: If an item in the list is missing the required identifier. + """ + if not identifier: + raise ValueError("Identifier cannot be empty.") + + identified_content = {} + for index, item in enumerate(content): + try: + val = item[identifier] + identified_content[val] = item + except (KeyError, TypeError): + raise KeyError( + f"Item at index {index} is missing the " + f"required identifier '{identifier}'. " + f"Item content: {item}" + ) from None + return identified_content + + +def _diff_list( + identifier: str, before_content: List[Any], after_content: List[Any] +) -> DiffResult: + """ + Diffs two lists of dictionaries based on a unique identifier. + + Args: + identifier: The key used to identify items across lists. + before_content: The original list. + after_content: The new list. + + Returns: + A dictionary with 'added', 'deleted', and 'modified' lists. + """ + # Transform list to dict using identifier as key + before_identified = _transform_identification(identifier, before_content) + after_identified = _transform_identification(identifier, after_content) + + results: DiffResult = {"added": [], "deleted": [], "modified": []} + + for key, val_after in after_identified.items(): + if key in before_identified: + val_before = before_identified[key] + if val_after != val_before: + results["modified"].append(val_after) # type: ignore + else: + results["added"].append(val_after) # type: ignore + + for key, val_before in before_identified.items(): + if key not in after_identified: + results["deleted"].append(val_before) # type: ignore + + return results + + +def _diff_dict( + identifier: str, + before_content: Dict[str, Any], + after_content: Dict[str, Any], +) -> DiffResult: + """ + Recursively diffs two dictionaries. + + Args: + identifier: The key used to identify items in nested lists. + before_content: The original dictionary. + after_content: The new dictionary. + + Returns: + A dictionary with 'added', 'deleted', and 'modified' sub-dictionaries. + """ + before_keys = before_content.keys() + after_keys = after_content.keys() + + added = {k: after_content[k] for k in after_keys - before_keys} + deleted = {k: before_content[k] for k in before_keys - after_keys} + modified: Dict[str, Any] = {} + + for k in before_keys & after_keys: + v_b, v_a = before_content[k], after_content[k] + + if v_b == v_a: + continue + + if isinstance(v_b, dict) and isinstance(v_a, dict): + diff_data = _diff_dict(identifier, v_b, v_a) + elif ( + isinstance(v_b, list) + and isinstance(v_a, list) + and ( + any(isinstance(i, dict) for i in v_b) + or any(isinstance(i, dict) for i in v_a) + ) + ): + diff_data = _diff_list(identifier, v_b, v_a) + else: + modified[k] = v_a + continue + + if diff_data["added"]: + added[k] = diff_data["added"] # type: ignore + if diff_data["deleted"]: + deleted[k] = diff_data["deleted"] # type: ignore + if diff_data["modified"]: + modified[k] = diff_data["modified"] # type: ignore + + # type: ignore + return {"added": added, "deleted": deleted, "modified": modified} + + +def diff( + before_content: JSONValue, after_content: JSONValue, identifier: str +) -> DiffResult: + """ + Main entry point for diffing two configurations. + + Args: + before_content: The original configuration (list or dict). + after_content: The new configuration (list or dict). + identifier: The key used to identify items in lists. + + Returns: + A DiffResult containing added, deleted, and modified changes. + + Raises: + ValueError: If inputs are lists but identifier is not provided. + TypeError: If contents are not of supported types or mixed incorrectly. + """ + # Handle dicts + if isinstance(before_content, dict) and isinstance(after_content, dict): + return _diff_dict(identifier, before_content, after_content) + + # Handle lists + if isinstance(before_content, list) and isinstance(after_content, list): + if not identifier: + raise ValueError("Identifier is needed to diff lists.") + return _diff_list(identifier, before_content, after_content) + + raise TypeError( + "Invalid contents to diff. Both must be lists or both must be dicts." + ) diff --git a/tools/apigee-config-diff/src/apigee_config_diff/diff/process.py b/tools/apigee-config-diff/src/apigee_config_diff/diff/process.py new file mode 100644 index 00000000..1d04d7d8 --- /dev/null +++ b/tools/apigee-config-diff/src/apigee_config_diff/diff/process.py @@ -0,0 +1,163 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import subprocess # nosec B404 + + +class MavenDeployer: + """Isolates the subprocess calls to execute Maven commands.""" + + def __init__(self, bearer=None, sa_path=None): + self.bearer = bearer + self.sa_path = sa_path + + def deploy(self, action: str, action_base_path: str, org: str, envs: set): + config_dir = os.path.join(action_base_path, org) + + if envs: + for e in envs: + print(f"Running env {e} for org {org}...") + self._run_mvn(action, config_dir, org, e) + else: + print(f"Running for org {org} (no specific env detected)...") + self._run_mvn(action, config_dir, org) + + def _run_mvn( + self, action: str, config_dir: str, org: str, env: str = None + ): + cmd = [ + "mvn", + "install", + f"-P{org}", + f"-Dapigee.config.dir={config_dir}", + f"-Dapigee.org={org}", + f"-Dapigee.config.options={action}", + ] + + if env: + cmd.append(f"-Dapigee.env={env}") + + if self.bearer: + cmd.append(f"-Dapigee.bearer={self.bearer}") + + if self.sa_path: + cmd.append(f"-Dapigee.serviceaccount.file={self.sa_path}") + + try: + subprocess.run(cmd, check=True) # nosec B603 + except subprocess.CalledProcessError as e: + print(f"Error running maven command: {e}", file=sys.stderr) + sys.exit(1) + + +def get_affected_orgs_and_envs( + action_base_path: str, affected_files: list +) -> dict: + """ + Parses the affected files to find the involved organizations + and environments. + Returns a dictionary mapping organization names to a set of + environment names. + """ + org_env_map = {} # org -> set(envs) + + for full_path in affected_files: + rel_path = os.path.relpath(full_path, action_base_path) + parts = rel_path.split(os.sep) + + org_name = parts[0] + + if not org_name or org_name == rel_path: + continue + + envs = org_env_map.setdefault(org_name, set()) + if len(parts) > 2 and parts[1] == "env": + envs.add(parts[2]) + + return org_env_map + + +def process_files( + output_base_path, resources_folder, confirm, bearer=None, sa_path=None +): + + actions = ["update", "delete"] + + # Strip trailing slash from resources_folder to ensure correct path joining + resources_subpath = resources_folder.strip(os.sep) + deployer = MavenDeployer(bearer, sa_path) + + for action in actions: + action_base_path = os.path.join( + output_base_path, action, resources_subpath + ) + + if not os.path.exists(action_base_path): + if os.path.exists(os.path.join(output_base_path, action)): + print( + f"No files found in {action_base_path} (subfolder empty)." + ) + continue + + print( + f"\nProcessing files in folder {action_base_path} " + f"for action {action}" + ) + + affected_files = [ + os.path.join(root, file) + for root, _, files in os.walk(action_base_path) + for file in files + ] + + if not affected_files: + print(f"No files to process for {action}.") + continue + + org_env_map = get_affected_orgs_and_envs( + action_base_path, affected_files + ) + + print("--- Affected Orgs and Environments ---") + if not org_env_map: + print("No orgs/environments found.") + else: + for org, envs in org_env_map.items(): + envs_list = list(envs) + print( + f"Org: {org}, Envs: {envs_list if envs_list else '(None)'}" + ) + + if confirm: + print( + f"--- Processing affected Orgs and Environments ({action}) ---" + ) + for org, envs in org_env_map.items(): + deployer.deploy(action, action_base_path, org, envs) + else: + print( + f"\n[Dry Run] The following structure would be processed " + f"for action ({action}):" + ) + try: + subprocess.run( # nosec B603 B607 + ["tree", action_base_path], check=False + ) + except FileNotFoundError: + for f in affected_files: + print(f" {f}") + + print("\nTo execute, call again with --confirm") diff --git a/tools/apigee-config-diff/src/apigee_config_diff/diff/util.py b/tools/apigee-config-diff/src/apigee_config_diff/diff/util.py new file mode 100644 index 00000000..a333f83e --- /dev/null +++ b/tools/apigee-config-diff/src/apigee_config_diff/diff/util.py @@ -0,0 +1,156 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import sys +import json +import subprocess # nosec B404 +from typing import Iterable + + +class GitClient: + """Encapsulates Git operations.""" + + @staticmethod + def resolve_commits( + commit_before: str, commit_after: str + ) -> tuple[str, str]: + previous_commit = "" + current_commit = commit_after + + if commit_before and commit_before == "0" * len(commit_before): + print( + "Previous commit is zero. Comparing against " + "an empty repository." + ) + return "", current_commit + + try: + git_rev_parse_proc = subprocess.run( # nosec B603 B607 + ["git", "rev-parse", "--verify", commit_before], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + text=True, + ) + except FileNotFoundError: + print( + "Error: 'git' command not found. Ensure it is " + "installed and in your PATH.", + file=sys.stderr, + ) + sys.exit(1) + + if git_rev_parse_proc.returncode == 0: + previous_commit = commit_before + else: + print( + f"Commit reference '{commit_before}' not found. " + "Listing all tracked files as 'added'." + ) + previous_commit = "" + + return previous_commit, current_commit + + @staticmethod + def read_file_contents(commit_hash: str, file_path: str) -> str: + return run_command_or_exit( + ["git", "show", f"{commit_hash}:{file_path}"], capture_output=True + ).stdout + + @staticmethod + def diff_hashes(hash_a: str, hash_b: str) -> subprocess.CompletedProcess: + return run_command_or_exit( + ["git", "diff", "--name-status", hash_a, hash_b], + capture_output=True, + ) + + @staticmethod + def list_files() -> subprocess.CompletedProcess: + return run_command_or_exit(["git", "ls-files"], capture_output=True) + + +def create_folder(folder_path): + if os.path.exists(folder_path): + shutil.rmtree(folder_path) + + os.makedirs(folder_path) + return folder_path + + +def find_resource_type(file_name: str, available_types: Iterable[str]): + for t in available_types: + if file_name.startswith(t): + return t + return None + + +def write_to_file(file_path, contents): + os.makedirs(os.path.dirname(file_path), exist_ok=True) + + json_str = json.dumps(contents, indent=4) + with open(file_path, "w") as f: + f.write(json_str) + + print(f"\nWrote {file_path} with contents:\n{json_str}") + + +def run_command_or_exit(cmd_args, capture_output=False, text=True, cwd=None): + stdout_setting = subprocess.PIPE if capture_output else None + stderr_setting = subprocess.PIPE + + try: + process = subprocess.run( # nosec B603 + cmd_args, + check=True, + text=text, + stdout=stdout_setting, + stderr=stderr_setting, + cwd=cwd, + ) + return process + except FileNotFoundError: + print( + f"Error: Command '{cmd_args[0]}' not found. " + "Ensure git is installed and in your PATH.", + file=sys.stderr, + ) + sys.exit(1) + except subprocess.CalledProcessError as e: + print(f"Error executing: {' '.join(e.cmd)}", file=sys.stderr) + print(f"Return code: {e.returncode}", file=sys.stderr) + + stdout_msg = ( + e.stdout.strip() if e.stdout and isinstance(e.stdout, str) else "" + ) + stderr_msg = ( + e.stderr.strip() if e.stderr and isinstance(e.stderr, str) else "" + ) + + if stdout_msg: + print(f"Stdout: {stdout_msg}", file=sys.stderr) + if stderr_msg: + print(f"Stderr: {stderr_msg}", file=sys.stderr) + sys.exit(e.returncode) + + +def merge(a, b): + if isinstance(a, dict) and isinstance(b, dict): + res = a.copy() + for k, v in b.items(): + res[k] = merge(res[k], v) if k in res else v + return res + if isinstance(a, list) and isinstance(b, list): + return a + b + return b if b is not None else a diff --git a/tools/apigee-config-diff/src/apigee_config_diff/main.py b/tools/apigee-config-diff/src/apigee_config_diff/main.py new file mode 100644 index 00000000..0dc41d19 --- /dev/null +++ b/tools/apigee-config-diff/src/apigee_config_diff/main.py @@ -0,0 +1,92 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +from .diff.check import detect_changes, write_temporary_files +from .diff.util import GitClient +from .diff.process import process_files + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Apigee Config Diff Generator and Deployer" + ) + + parser.add_argument( + "--commit-before", + help="Previous commit hash (default: HEAD~1)", + default="HEAD~1", + ) + parser.add_argument( + "--current-commit", + help="Current commit hash (default: HEAD)", + default="HEAD", + ) + parser.add_argument( + "--folder", + help="Files folder from repo root to diff (default: resources)", + default="resources", + ) + parser.add_argument( + "--output", + help="Output folder for generated trees (default: output)", + default="output", + ) + parser.add_argument( + "--confirm", + action="store_true", + help="Execute the Maven plugin (apply changes)", + ) + parser.add_argument( + "--bearer", help="Apigee bearer token (optional)", default=None + ) + parser.add_argument( + "--sa-path", + help="Path to service account key file (optional)", + default=None, + ) + + return parser.parse_args() + + +def main(): + args = parse_args() + + previous_commit, current_commit = GitClient.resolve_commits( + args.commit_before, args.current_commit + ) + + # Find files added, deleted or modified + added_files, deleted_files, modified_files = detect_changes( + previous_commit, current_commit, args.folder + ) + + # Write the files to be processed + write_temporary_files( + added_files, + deleted_files, + modified_files, + previous_commit, + current_commit, + args.output, + ) + + # Process (Plan or Apply) + process_files( + args.output, args.folder, args.confirm, args.bearer, args.sa_path + ) + + +if __name__ == "__main__": # pragma: no cover + main() diff --git a/tools/apigee-config-diff/tests/test_check.py b/tools/apigee-config-diff/tests/test_check.py new file mode 100644 index 00000000..3dd57459 --- /dev/null +++ b/tools/apigee-config-diff/tests/test_check.py @@ -0,0 +1,318 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import patch, MagicMock +import os + +from apigee_config_diff.diff.check import detect_changes, write_temporary_files + + +@patch("apigee_config_diff.diff.check.GitClient.diff_hashes") +def test_detect_changes_with_previous_commit(mock_git_diff_hashes): + mock_result = MagicMock() + mock_result.stdout = ( + "M\tresources/file1.json\n" + "A\tresources/file2.json\n" + "D\tresources/file3.json\n" + "R100\tresources/old_name.json\tresources/new_name.json\n" + "M\tother/ignored.txt" + ) + mock_git_diff_hashes.return_value = mock_result + + previous_commit = "abc1234" + current_commit = "def5678" + resources_base_path = "resources/" + + added, deleted, modified = detect_changes( + previous_commit, current_commit, resources_base_path + ) + + assert "resources/file2.json" in added + assert "resources/new_name.json" in added + assert "resources/file3.json" in deleted + assert "resources/old_name.json" in deleted + assert "resources/file1.json" in modified + assert "other/ignored.txt" not in (added + deleted + modified) + assert len(added) == 2 + assert len(deleted) == 2 + assert len(modified) == 1 + + +@patch("apigee_config_diff.diff.check.GitClient.list_files") +def test_detect_changes_initial_commit(mock_run_command_or_exit): + mock_result = MagicMock() + mock_result.stdout = ( + "resources/file1.json\n" + "resources/file2.json\n" + "resources/file3.json" + ) + mock_run_command_or_exit.return_value = mock_result + + previous_commit = None + current_commit = "def5678" + resources_base_path = "resources/" + + added, deleted, modified = detect_changes( + previous_commit, current_commit, resources_base_path + ) + + assert "resources/file1.json" in added + assert "resources/file2.json" in added + assert "resources/file3.json" in added + assert len(added) == 3 + assert len(deleted) == 0 + assert len(modified) == 0 + + +@patch("apigee_config_diff.diff.check.write_to_file") +@patch("apigee_config_diff.diff.check.GitClient.read_file_contents") +@patch("apigee_config_diff.diff.check.create_folder") +@patch("apigee_config_diff.diff.check.find_resource_type") +@patch("apigee_config_diff.diff.check.diff") +@patch.dict("apigee_config_diff.diff.check.RESOURCES_ID", {"some_type": "key"}) +def test_write_temporary_files_basic( + mock_diff_func, + mock_find_resource_type, + mock_create_folder, + mock_read_git_contents, + mock_write_to_file, +): + mock_create_folder.side_effect = lambda x: x + mock_read_git_contents.return_value = "{}" + mock_find_resource_type.return_value = "some_type" + mock_diff_elements = { + "added": [{"name": "new_item_from_diff"}], + "modified": [{"name": "mod_item_from_diff"}], + "deleted": [{"name": "del_item_from_diff"}], + } + mock_diff_func.return_value = mock_diff_elements + + added_files = ["resources/newly_added_file.json"] + deleted_files = ["resources/to_be_deleted_file.json"] + modified_files = ["resources/modified_file.json"] + previous_commit = "abc1234" + current_commit = "def5678" + tmp_base_path = "./tmp/test_output" + + update_folder_path = os.path.join(tmp_base_path, "update") + delete_folder_path = os.path.join(tmp_base_path, "delete") + + write_temporary_files( + added_files, + deleted_files, + modified_files, + previous_commit, + current_commit, + tmp_base_path, + ) + + mock_create_folder.assert_any_call(update_folder_path) + mock_create_folder.assert_any_call(delete_folder_path) + + path_for_added_file = os.path.join( + update_folder_path, "resources/newly_added_file.json" + ) + mock_read_git_contents.assert_any_call( + current_commit, "resources/newly_added_file.json" + ) + mock_write_to_file.assert_any_call(path_for_added_file, {}) + + path_for_deleted_file = os.path.join( + delete_folder_path, "resources/to_be_deleted_file.json" + ) + mock_read_git_contents.assert_any_call( + previous_commit, "resources/to_be_deleted_file.json" + ) + mock_write_to_file.assert_any_call(path_for_deleted_file, {}) + + mod_f_path = "resources/modified_file.json" + mock_read_git_contents.assert_any_call(previous_commit, mod_f_path) + mock_read_git_contents.assert_any_call(current_commit, mod_f_path) + mock_diff_func.assert_any_call({}, {}, "key") + + expected_content_for_update_from_mod = ( + mock_diff_elements["added"] + mock_diff_elements["modified"] + ) + path_for_mod_update = os.path.join(update_folder_path, mod_f_path) + mock_write_to_file.assert_any_call( + path_for_mod_update, expected_content_for_update_from_mod + ) + + path_for_mod_delete = os.path.join( + delete_folder_path, f"{mod_f_path}.delete" + ) + mock_write_to_file.assert_any_call( + path_for_mod_delete, mock_diff_elements["deleted"] + ) + + +@patch("apigee_config_diff.diff.check.write_to_file") +@patch("apigee_config_diff.diff.check.GitClient.read_file_contents") +@patch("apigee_config_diff.diff.check.create_folder") +@patch("apigee_config_diff.diff.check.find_resource_type") +@patch("apigee_config_diff.diff.check.diff") +@patch.dict( + "apigee_config_diff.diff.check.RESOURCES_ID", {"developerApps": "name"} +) +def test_write_temporary_files_dict_merge_logic( + mock_diff_func, + mock_find_resource_type, + mock_create_folder, + mock_read_git_contents, + mock_write_to_file, +): + mock_create_folder.side_effect = lambda x: x + mock_read_git_contents.return_value = "{}" + mock_find_resource_type.return_value = "developerApps" + + # Simulate the "overlapping key" scenario where dev@example.com + # has an added AND modified app + mock_diff_elements = { + "added": {"dev@example.com": [{"name": "App_B"}]}, + "modified": {"dev@example.com": [{"name": "App_A"}]}, + "deleted": {}, + } + mock_diff_func.return_value = mock_diff_elements + + modified_files = ["resources/developerApps.json"] + tmp_base_path = "./tmp/test_output" + update_folder_path = os.path.join(tmp_base_path, "update") + + write_temporary_files( + [], [], modified_files, "prev", "curr", tmp_base_path + ) + + expected_merged_content = { + "dev@example.com": [{"name": "App_B"}, {"name": "App_A"}] + } + + path_for_mod_update = os.path.join( + update_folder_path, "resources/developerApps.json" + ) + + # Verify that the merged content contains BOTH apps, not just the last one + mock_write_to_file.assert_any_call( + path_for_mod_update, expected_merged_content + ) + + +@patch("apigee_config_diff.diff.check.GitClient.diff_hashes") +def test_detect_changes_edge_cases(mock_git_diff_hashes): + mock_result = MagicMock() + mock_result.stdout = ( + "R100\tresources/old_rename.json\n" # Missing path_new + "C075\tresources/copied_fail.json\n" # Missing path_new + "X\tresources/unknown.json\n" # Unknown status + "C075\tresources/old.json\tresources/new.json" # Valid copy + ) + mock_git_diff_hashes.return_value = mock_result + + added, deleted, modified = detect_changes("a", "b", "resources") + + assert "resources/new.json" in added + assert len(added) == 1 + assert len(deleted) == 0 + + # Test unknown status and path_new is None warnings + # (implicitly covered by calling detect_changes with these mocks) + # We can also check if they don't crash + # The current code prints to stderr + detect_changes("a", "b", "resources/") + + +@patch("apigee_config_diff.diff.check.write_to_file") +@patch("apigee_config_diff.diff.check.GitClient.read_file_contents") +@patch("apigee_config_diff.diff.check.create_folder") +@patch("apigee_config_diff.diff.check.find_resource_type") +def test_write_temporary_files_unknown_type( + mock_find_resource_type, + mock_create_folder, + mock_read_git_contents, + mock_write_to_file, +): + mock_create_folder.side_effect = lambda x: x + mock_read_git_contents.return_value = '{"full": "content"}' + mock_find_resource_type.return_value = None + + modified_files = ["resources/unknown.json"] + write_temporary_files([], [], modified_files, "prev", "curr", "./tmp") + + mock_write_to_file.assert_any_call( + "./tmp/update/resources/unknown.json", {"full": "content"} + ) + + +@patch("apigee_config_diff.diff.check.GitClient.list_files") +def test_detect_changes_initial_commit_no_files(mock_run_command_or_exit): + mock_result = MagicMock() + mock_result.stdout = "" + mock_run_command_or_exit.return_value = mock_result + + added, deleted, modified = detect_changes(None, "def5678", "resources/") + assert len(added) == 0 + + +@patch("apigee_config_diff.diff.check.GitClient.list_files") +def test_detect_changes_initial_commit_empty_line(mock_run_command_or_exit): + mock_result = MagicMock() + mock_result.stdout = "\n" + mock_run_command_or_exit.return_value = mock_result + + added, deleted, modified = detect_changes(None, "def5678", "resources/") + assert len(added) == 0 + + +@patch("apigee_config_diff.diff.check.write_to_file") +@patch("apigee_config_diff.diff.check.GitClient.read_file_contents") +@patch("apigee_config_diff.diff.check.create_folder") +@patch("apigee_config_diff.diff.check.find_resource_type") +@patch("apigee_config_diff.diff.check.diff") +@patch.dict("apigee_config_diff.diff.check.RESOURCES_ID", {"some_type": "key"}) +def test_write_temporary_files_empty_diff( + mock_diff_func, + mock_find_resource_type, + mock_create_folder, + mock_read_git_contents, + mock_write_to_file, +): + mock_create_folder.side_effect = lambda x: x + mock_read_git_contents.return_value = "{}" + mock_find_resource_type.return_value = "some_type" + mock_diff_func.return_value = {"added": [], "modified": [], "deleted": []} + + write_temporary_files( + [], [], ["resources/file.json"], "prev", "curr", "./tmp" + ) + # write_to_file should NOT be called for update/delete if they are empty + assert mock_write_to_file.call_count == 0 + + +@patch("apigee_config_diff.diff.check.GitClient.read_file_contents") +def test_calculate_file_diffs_invalid_json(mock_read_git_contents): + mock_read_git_contents.return_value = "{ invalid json" + + from apigee_config_diff.diff.check import calculate_file_diffs + + added_files = ["resources/added.json"] + deleted_files = ["resources/deleted.json"] + modified_files = ["resources/modified.json"] + + files_to_update, files_to_delete = calculate_file_diffs( + added_files, deleted_files, modified_files, "prev", "curr" + ) + + # Should safely skip the invalid json files and not crash + assert "resources/added.json" not in files_to_update + assert "resources/deleted.json" not in files_to_delete + assert "resources/modified.json" not in files_to_update diff --git a/tools/apigee-config-diff/tests/test_diff.py b/tools/apigee-config-diff/tests/test_diff.py new file mode 100644 index 00000000..643037e5 --- /dev/null +++ b/tools/apigee-config-diff/tests/test_diff.py @@ -0,0 +1,197 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from apigee_config_diff.diff.diff import _transform_identification, diff + + +def test_identify_object(): + input_data = [ + {"id": 1, "val": 11}, + {"id": 2, "val": 22}, + {"id": 3, "val": 33}, + ] + + expected = { + 1: {"id": 1, "val": 11}, + 2: {"id": 2, "val": 22}, + 3: {"id": 3, "val": 33}, + } + + transformed = _transform_identification("id", input_data) + assert transformed == expected + + +def test_identify_object_unmapped(): + input_data = [{"id": 1, "val": 11}] + with pytest.raises(ValueError, match="Identifier cannot be empty."): + _transform_identification("", input_data) + + +def test_transform_identification_missing_key(): + input_data = [{"id": 1, "val": 11}, {"val": 22}] + with pytest.raises( + KeyError, + match="Item at index 1 is missing the required identifier 'id'", + ): + _transform_identification("id", input_data) + + +def test_diff_list(): + before_data = [ + {"key": "a", "val": 1}, + {"key": "b", "val": 2}, + {"key": "z", "val": 9}, + ] + after_data = [ + {"key": "a", "val": 11}, + {"key": "c", "val": 3}, + {"key": "d", "val": 4}, + {"key": "z", "val": 9}, + ] + + result = diff(before_data, after_data, "key") + + assert sorted(result["added"], key=lambda x: x["key"]) == [ + {"key": "c", "val": 3}, + {"key": "d", "val": 4}, + ] + assert result["deleted"] == [{"key": "b", "val": 2}] + assert result["modified"] == [{"key": "a", "val": 11}] + + +def test_diff_dict(): + before_data = { + "first": [{"key": "a", "val": 1}, {"key": "b", "val": 2}], + "second": [{"key": "a", "val": 1}, {"key": "b", "val": 2}], + "third": [{"key": "a", "val": 1}, {"key": "b", "val": 2}], + } + after_data = { + "first": [{"key": "a", "val": 11}, {"key": "c", "val": 3}], + "fourth": [{"key": "a", "val": 1}, {"key": "f", "val": 4}], + "third": [{"key": "b", "val": 22}], + } + + result = diff(before_data, after_data, "key") + + assert result["added"] == { + "first": [{"key": "c", "val": 3}], + "fourth": [{"key": "a", "val": 1}, {"key": "f", "val": 4}], + } + assert result["deleted"] == { + "first": [{"key": "b", "val": 2}], + "second": [{"key": "a", "val": 1}, {"key": "b", "val": 2}], + "third": [{"key": "a", "val": 1}], + } + assert result["modified"] == { + "first": [{"key": "a", "val": 11}], + "third": [{"key": "b", "val": 22}], + } + + +def test_diff_invalid_types(): + with pytest.raises(TypeError, match="Invalid contents to diff."): + diff([1], {"a": 1}, "id") + + +def test_diff_list_no_identifier(): + with pytest.raises( + ValueError, match="Identifier is needed to diff lists." + ): + diff([], [], "") + + +def test_transform_identification_no_identifier(): + with pytest.raises(ValueError, match="Identifier cannot be empty."): + _transform_identification("", []) + + +def test_diff_dict_nested_dict(): + before_data = {"outer": {"inner_list": [{"key": "a", "val": 1}]}} + after_data = { + "outer": { + "inner_list": [{"key": "a", "val": 2}, {"key": "b", "val": 3}] + } + } + result = diff(before_data, after_data, "key") + assert result["modified"] == { + "outer": {"inner_list": [{"key": "a", "val": 2}]} + } + assert result["added"] == { + "outer": {"inner_list": [{"key": "b", "val": 3}]} + } + assert result["deleted"] == {} + + +def test_diff_dict_primitive_values(): + before_data = { + "primitive_modified": "old", + "primitive_same": "same", + "primitive_list_to_str": ["list"], + } + after_data = { + "primitive_modified": "new", + "primitive_same": "same", + "primitive_list_to_str": "str", + } + result = diff(before_data, after_data, "key") + assert result["modified"]["primitive_modified"] == "new" + assert "primitive_same" not in result["modified"] + assert result["modified"]["primitive_list_to_str"] == "str" + + +def test_diff_dict_mixed_types(): + # Test list replaced by dict + before_data = {"key": [1, 2, 3]} + after_data = {"key": {"a": 1}} + result = diff(before_data, after_data, "id") + assert result["modified"]["key"] == {"a": 1} + assert result["added"] == {} + assert result["deleted"] == {} + + # Test dict replaced by list + before_data = {"key": {"a": 1}} + after_data = {"key": [1, 2, 3]} + result = diff(before_data, after_data, "id") + assert result["modified"]["key"] == [1, 2, 3] + + +def test_transform_identification_type_error(): + # Test when item is not a dict + input_data = ["not a dict"] + with pytest.raises( + KeyError, + match="Item at index 0 is missing the required identifier 'id'", + ): + _transform_identification("id", input_data) + + +def test_diff_dict_nested_dict_deleted(): + before_data = {"outer": {"key1": "val1", "key2": "val2"}} + after_data = {"outer": {"key1": "val1"}} + result = diff(before_data, after_data, "id") + assert result["deleted"] == {"outer": {"key2": "val2"}} + + assert result["added"] == {} + assert result["modified"] == {} + + +def test_diff_dict_list_of_primitives(): + """Test that lists of primitives are handled correctly.""" + before_data = {"env": ["test", "prod"], "other": ["same"]} + after_data = {"env": ["test", "dev"], "other": ["same"]} + result = diff(before_data, after_data, "name") + assert result["modified"] == {"env": ["test", "dev"]} + assert result["added"] == {} + assert result["deleted"] == {} diff --git a/tools/apigee-config-diff/tests/test_integration_main.py b/tools/apigee-config-diff/tests/test_integration_main.py new file mode 100644 index 00000000..33662f31 --- /dev/null +++ b/tools/apigee-config-diff/tests/test_integration_main.py @@ -0,0 +1,778 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import patch, MagicMock +from apigee_config_diff.main import main +import json +import os + + +@patch( + "sys.argv", + [ + "main.py", + "--commit-before", + "previous_commit", + "--current-commit", + "current_commit", + "--folder", + "resources/", + "--output", + "./tmp/apigee", + ], +) +@patch("apigee_config_diff.diff.check.GitClient.diff_hashes") +@patch("apigee_config_diff.diff.check.GitClient.read_file_contents") +@patch("subprocess.run") +def test_write_temporary_files_basic( + mock_subprocess_run, mock_read_git_contents, mock_git_diff_hashes +): + mock_subprocess_run.return_value.returncode = 0 + + # Mock the diff files + mock_git_diff_hashes.return_value = _mock_git_diff() + + # Mock the file contents + mock_read_git_contents.side_effect = _mock_git_file_content + + main() + + # Check that update/delete directories were created and populated + update_dir = "./tmp/apigee/update/resources/my-org/env/dev/" + delete_dir = "./tmp/apigee/delete/resources/my-org/env/dev/" + + assert os.path.exists(os.path.join(update_dir, "flowhooks.json")) + assert os.path.exists(os.path.join(update_dir, "flowhooks-added.json")) + assert os.path.exists(os.path.join(delete_dir, "flowhooks-old.json")) + + # Validate content merging correctly + with open( + "./tmp/apigee/update/resources/my-org/org/developerApps.json" + ) as f: + dev_apps = json.load(f) + assert "hugh@example.com" in dev_apps + assert "hughnew@example.com" in dev_apps + assert dev_apps["hugh@example.com"][0]["name"] == "hughapp" + assert ( + dev_apps["hugh@example.com"][0]["callbackUrl"] + == "http://weatherappModified.com" + ) + + with open( + "./tmp/apigee/update/resources/my-org/env/dev/targetServers.json" + ) as f: + target_servers = json.load(f) + assert len(target_servers) == 1 + assert target_servers[0]["name"] == "Enterprisetarget" + assert target_servers[0]["isEnabled"] is False + + +def _mock_git_diff(): + mock_result = MagicMock() + + mock_result.stdout = ( + "M\tresources/my-org/env/dev/flowhooks.json\n" + "A\tresources/my-org/env/dev/flowhooks-added.json\n" + "D\tresources/my-org/env/dev/flowhooks-old.json\n" + "M\tresources/my-org/env/dev/references.json\n" + "A\tresources/my-org/env/dev/references-added.json\n" + "D\tresources/my-org/env/dev/references-old.json\n" + "M\tresources/my-org/env/dev/targetServers.json\n" + "A\tresources/my-org/env/dev/targetServers-added.json\n" + "D\tresources/my-org/env/dev/targetServers-old.json\n" + "M\tresources/my-org/env/dev/keystores.json\n" + "A\tresources/my-org/env/dev/keystores-added.json\n" + "D\tresources/my-org/env/dev/keystores-old.json\n" + "M\tresources/my-org/env/dev/aliases.json\n" + "A\tresources/my-org/env/dev/aliases-added.json\n" + "D\tresources/my-org/env/dev/aliases-old.json\n" + "M\tresources/my-org/org/apiProducts.json\n" + "A\tresources/my-org/org/apiProducts-added.json\n" + "D\tresources/my-org/org/apiProducts-old.json\n" + "M\tresources/my-org/org/developers.json\n" + "A\tresources/my-org/org/developers-added.json\n" + "D\tresources/my-org/org/developers-old.json\n" + "M\tresources/my-org/org/developerApps.json\n" + "A\tresources/my-org/org/developerApps-added.json\n" + "D\tresources/my-org/org/developerApps-old.json" + ) + + return mock_result + + +file_contents = { + "previous_commit": { + "resources/my-org/env/dev/flowhooks.json": """ + [ + { + "flowHookPoint":"PreProxyFlowHook", + "sharedFlow":"test" + } + ] + """, + "resources/my-org/env/dev/flowhooks-old.json": """ + [ + { + "flowHookPoint":"PreTargetFlowHook", + "sharedFlow":"test" + } + ] + """, + "resources/my-org/env/dev/references.json": """ + [ + { + "name" : "sampleReference", + "refers": "testKeyStorename", + "resourceType": "KeyStore" + } + ] + """, + "resources/my-org/env/dev/references-old.json": """ + [ + { + "name" : "oldReference", + "refers": "testKeyStorename", + "resourceType": "KeyStore" + } + ] + """, + "resources/my-org/env/dev/targetServers.json": """ + [ + { + "name": "Enterprisetarget", + "host": "example.com", + "isEnabled": true, + "port": 8080 + }, + { + "name": "ESBTarget", + "host": "enterprise.com", + "isEnabled": true, + "port": 8080, + "sSLInfo": { + "clientAuthEnabled": "false", + "enabled": "true", + "ignoreValidationErrors": "false", + "keyAlias": "key_alias", + "keyStore": "keystore_name", + "trustStore": "truststore_name" + } + } + ] + """, + "resources/my-org/env/dev/targetServers-old.json": """ + [ + { + "name": "oldTarget", + "host": "old.example.com", + "isEnabled": true, + "port": 1111 + } + ] + """, + "resources/my-org/env/dev/keystores.json": """ + [ + { + "name" : "testKeyStorename" + } + ] + """, + "resources/my-org/env/dev/keystores-old.json": """ + [ + { + "name" : "oldKeyStorename" + } + ] + """, + "resources/my-org/env/dev/aliases.json": """ + [ + { + "alias":"testSelfSignedCert", + "keystorename": "testKeyStorename", + "format": "selfsignedcert", + "keySize":"2048", + "sigAlg":"SHA256withRSA", + "subject":{ + "commonName":"testcommonName" + }, + "certValidityInDays":"90" + }, + { + "alias":"testAliasCertFile", + "keystorename": "testKeyStorename", + "ignoreExpiryValidation": true, + "format": "keycertfile", + "certFilePath":"./tmp/certs/keystore.pem" + }, + { + "alias":"testAliasKeyCertFileAndKey", + "keystorename": "testKeyStorename", + "ignoreExpiryValidation": true, + "format": "keycertfile", + "certFilePath":"./tmp/certs/keystore.pem", + "keyFilePath":"./tmp/certs/keystore.key", + "password":"dummy" + }, + { + "alias":"testAliasPKCS12", + "keystorename": "testKeyStorename", + "ignoreExpiryValidation": true, + "format": "pkcs12", + "filePath":"./tmp/certs/myKeystore.p12", + "password":"dummy" + } + ] + """, + "resources/my-org/env/dev/aliases-old.json": """ + [ + { + "alias":"oldSelfSignedCert", + "keystorename": "testKeyStorename", + "format": "selfsignedcert", + "keySize":"2048", + "sigAlg":"SHA256withRSA", + "subject":{ + "commonName":"testcommonName" + }, + "certValidityInDays":"90" + } + ] + """, + "resources/my-org/org/apiProducts.json": """ + [ + { + "name":"weatherProduct", + "displayName":"weatherProduct", + "description":"weatherProduct", + "approvalType":"auto", + "environments":[ + "test" + ], + "attributes": [ + { + "name": "access", + "value": "public" + } + ], + "quota":"10000", + "quotaInterval":"1", + "quotaTimeUnit":"month", + "operationGroup":{ + "operationConfigs":[ + { + "apiSource":"forecastweatherapi", + "operations":[ + { + "resource":"/", + "methods":[ + "GET" + ] + } + ], + "quota":{ + "limit":"1000", + "interval":"1", + "timeUnit":"month" + }, + "attributes":[ + { + "name":"foo", + "value":"bar" + } + ] + } + ] + } + }, + { + "name":"weatherProduct-legacy", + "displayName":"weatherProduct-legacy", + "description":"weatherProduct-legacy", + "apiResources":[ + "/**", + "/" + ], + "approvalType":"auto", + "attributes":[ + { + "name":"description", + "value":"weatherProduct-legacy" + }, + { + "name": "access", + "value": "public" + }, + { + "name":"developer.quota.limit", + "value":"10000" + }, + { + "name":"developer.quota.interval", + "value":"1" + }, + { + "name":"developer.quota.timeunit", + "value":"month" + } + ], + "environments":[ + "test" + ], + "proxies":[ + "forecastweatherapi" + ], + "quota":"10000", + "quotaInterval":"1", + "quotaTimeUnit":"month", + "scopes":[] + } + ] + """, + "resources/my-org/org/apiProducts-old.json": """ + [ + { + "name":"oldWeatherProduct-legacy", + "displayName":"weatherProduct-old", + "description":"weatherProduct-old", + "apiResources":[ + "/**", + "/" + ], + "approvalType":"auto", + "attributes":[ + { + "name":"description", + "value":"weatherProduct-legacy" + }, + { + "name": "access", + "value": "public" + }, + { + "name":"developer.quota.limit", + "value":"10000" + }, + { + "name":"developer.quota.interval", + "value":"1" + }, + { + "name":"developer.quota.timeunit", + "value":"month" + } + ], + "environments":[ + "old" + ], + "proxies":[ + "oldproxy" + ], + "quota":"10000", + "quotaInterval":"1", + "quotaTimeUnit":"month", + "scopes":[] + } + ] + """, + "resources/my-org/org/developers.json": """ + [ + { + "attributes": [], + "email": "hugh@example.com", + "firstName": "Hugh", + "lastName": "Jack", + "userName": "hughexample" + } + ] + """, + "resources/my-org/org/developers-old.json": """ + [ + { + "attributes": [], + "email": "old@example.com", + "firstName": "Hugh", + "lastName": "Old", + "userName": "hughold" + } + ] + """, + "resources/my-org/org/developerApps.json": """ + { + "hugh@example.com": [ + { + "apiProducts": [ + "weatherProduct" + ], + "callbackUrl": "http://weatherapp.com", + "name": "hughapp", + "scopes": [] + } + ] + } + """, + "resources/my-org/org/developerApps-old.json": """ + { + "hughold@example.com": [ + { + "apiProducts": [ + "weatherProductOld" + ], + "callbackUrl": "http://weatherapp.com", + "name": "hughappold", + "scopes": [] + } + ] + } + """, + }, + "current_commit": { + "resources/my-org/env/dev/flowhooks.json": """ + [ + { + "flowHookPoint":"PreProxyFlowHook", + "sharedFlow":"test-modified" + } + ] + """, + "resources/my-org/env/dev/flowhooks-added.json": """ + [ + { + "flowHookPoint":"PostTargetFlowHook", + "sharedFlow":"test-added" + } + ] + """, + "resources/my-org/env/dev/references.json": """ + [ + { + "name" : "sampleReference", + "refers": "anotherKeyStorename", + "resourceType": "KeyStore" + } + ] + """, + "resources/my-org/env/dev/references-added.json": """ + [ + { + "name" : "NewReference", + "refers": "newKeyStorename", + "resourceType": "KeyStore" + } + ] + """, + "resources/my-org/env/dev/targetServers.json": """ + [ + { + "name": "Enterprisetarget", + "host": "example.com", + "isEnabled": false, + "port": 8081 + }, + { + "name": "ESBTarget", + "host": "enterprise.com", + "isEnabled": true, + "port": 8080, + "sSLInfo": { + "clientAuthEnabled": "false", + "enabled": "true", + "ignoreValidationErrors": "false", + "keyAlias": "key_alias", + "keyStore": "keystore_name", + "trustStore": "truststore_name" + } + } + ] + """, + "resources/my-org/env/dev/targetServers-added.json": """ + [ + { + "name": "NewTarget", + "host": "new.com", + "isEnabled": true, + "port": 8080 + } + ] + """, + "resources/my-org/env/dev/keystores.json": """ + [ + { + "name" : "modifiedKeyStorename" + } + ] + """, + "resources/my-org/env/dev/keystores-added.json": """ + [ + { + "name" : "newKeyStorename" + } + ] + """, + "resources/my-org/env/dev/aliases.json": """ + [ + { + "alias":"testSelfSignedCert", + "keystorename": "modifiedKeyStorename", + "format": "selfsignedcert", + "keySize":"2048", + "sigAlg":"SHA256withRSA", + "subject":{ + "commonName":"testcommonName" + }, + "certValidityInDays":"90" + }, + { + "alias":"testAliasCertFile", + "keystorename": "testKeyStorename", + "ignoreExpiryValidation": true, + "format": "keycertfile", + "certFilePath":"./tmp/certs/keystore.pem" + }, + { + "alias":"testAliasKeyCertFileAndKey", + "keystorename": "anotherModifiedKeyStorename", + "ignoreExpiryValidation": true, + "format": "keycertfile", + "certFilePath":"./tmp/certs/keystore.pem", + "keyFilePath":"./tmp/certs/keystore.key", + "password":"dummy" + }, + { + "alias":"testAliasPKCS12", + "keystorename": "testKeyStorename", + "ignoreExpiryValidation": true, + "format": "pkcs12", + "filePath":"./tmp/certs/myKeystore.p12", + "password":"dummy" + } + ] + """, + "resources/my-org/env/dev/aliases-added.json": """ + [ + { + "alias":"newSelfSignedCert", + "keystorename": "testKeyStorename", + "format": "selfsignedcert", + "keySize":"2048", + "sigAlg":"SHA256withRSA", + "subject":{ + "commonName":"testcommonName" + }, + "certValidityInDays":"90" + } + ] + """, + "resources/my-org/org/apiProducts.json": """ + [ + { + "name":"weatherProduct", + "displayName":"modifiedWeatherProduct", + "description":"weatherProduct", + "approvalType":"auto", + "environments":[ + "test" + ], + "attributes": [ + { + "name": "access", + "value": "public" + } + ], + "quota":"7000", + "quotaInterval":"1", + "quotaTimeUnit":"month", + "operationGroup":{ + "operationConfigs":[ + { + "apiSource":"forecastweatherapi", + "operations":[ + { + "resource":"/", + "methods":[ + "GET" + ] + } + ], + "quota":{ + "limit":"1000", + "interval":"1", + "timeUnit":"month" + }, + "attributes":[ + { + "name":"foo", + "value":"bar" + } + ] + } + ] + } + }, + { + "name":"weatherProduct-legacy", + "displayName":"weatherProduct-legacy", + "description":"weatherProduct-legacy", + "apiResources":[ + "/**", + "/" + ], + "approvalType":"auto", + "attributes":[ + { + "name":"description", + "value":"weatherProduct-legacy" + }, + { + "name": "access", + "value": "public" + }, + { + "name":"developer.quota.limit", + "value":"10000" + }, + { + "name":"developer.quota.interval", + "value":"1" + }, + { + "name":"developer.quota.timeunit", + "value":"month" + } + ], + "environments":[ + "test" + ], + "proxies":[ + "forecastweatherapi" + ], + "quota":"10000", + "quotaInterval":"1", + "quotaTimeUnit":"month", + "scopes":[] + } + ] + """, + "resources/my-org/org/apiProducts-added.json": """ + [ + { + "name":"aNewProduct", + "displayName":"newProduct", + "description":"aNewProduct", + "apiResources":[ + "/**", + "/" + ], + "approvalType":"auto", + "attributes":[ + { + "name":"description", + "value":"weatherProduct-legacy" + }, + { + "name": "access", + "value": "public" + }, + { + "name":"developer.quota.limit", + "value":"10000" + }, + { + "name":"developer.quota.interval", + "value":"1" + }, + { + "name":"developer.quota.timeunit", + "value":"month" + } + ], + "environments":[ + "test" + ], + "proxies":[ + "forecastweatherapi" + ], + "quota":"10000", + "quotaInterval":"1", + "quotaTimeUnit":"month", + "scopes":[] + } + ] + """, + "resources/my-org/org/developers.json": """ + [ + { + "attributes": [], + "email": "hugh@example.com", + "firstName": "Hugh", + "lastName": "Jack Modified", + "userName": "hughexample" + } + ] + """, + "resources/my-org/org/developers-added.json": """ + [ + { + "attributes": [], + "email": "new@example.com", + "firstName": "New", + "lastName": "Dev", + "userName": "newexample" + } + ] + """, + "resources/my-org/org/developerApps.json": """ + { + "hugh@example.com": [ + { + "apiProducts": [ + "weatherProductModified" + ], + "callbackUrl": "http://weatherappModified.com", + "name": "hughapp", + "scopes": [] + } + ], + + "hughnew@example.com": [ + { + "apiProducts": [ + "weatherProductNew" + ], + "callbackUrl": "http://weatherappNew.com", + "name": "hughappNew", + "scopes": [] + } + ] + } + """, + "resources/my-org/org/developerApps-added.json": """ + { + "hughadded@example.com": [ + { + "apiProducts": [ + "weatherProductNew" + ], + "callbackUrl": "http://weatherappNew.com", + "name": "hughappAdded", + "scopes": [] + } + ] + } + """, + }, +} + + +def _mock_git_file_content(commit_hash, f_path): + return file_contents[commit_hash][f_path] diff --git a/tools/apigee-config-diff/tests/test_main.py b/tools/apigee-config-diff/tests/test_main.py new file mode 100644 index 00000000..34084f88 --- /dev/null +++ b/tools/apigee-config-diff/tests/test_main.py @@ -0,0 +1,31 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import patch +from apigee_config_diff.main import main + + +def test_main_execution(): + with patch( + "sys.argv", + ["main.py", "--commit-before", "HEAD~1", "--current-commit", "HEAD"], + ), patch( + "apigee_config_diff.diff.check.detect_changes", + return_value=([], [], []), + ), patch( + "apigee_config_diff.diff.check.write_temporary_files" + ), patch( + "apigee_config_diff.diff.process.process_files" + ): + main() diff --git a/tools/apigee-config-diff/tests/test_process.py b/tools/apigee-config-diff/tests/test_process.py new file mode 100644 index 00000000..a3266ddb --- /dev/null +++ b/tools/apigee-config-diff/tests/test_process.py @@ -0,0 +1,186 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import pytest +from unittest.mock import patch +from apigee_config_diff.diff.process import process_files + + +def test_process_files_dry_run(tmp_path, capsys): + # Create a mock structure + # update/resources/org1/env/dev/file.json + # update/resources/org1/org/file.json + update_dir = tmp_path / "update" / "resources" + update_dir.mkdir(parents=True) + (update_dir / "org1").mkdir() + (update_dir / "org1" / "env").mkdir() + (update_dir / "org1" / "env" / "dev").mkdir() + (update_dir / "org1" / "env" / "dev" / "test.json").write_text("{}") + (update_dir / "org1" / "org").mkdir() + (update_dir / "org1" / "org" / "products.json").write_text("{}") + + with patch("subprocess.run") as mock_run: + process_files(str(tmp_path), "resources", confirm=False) + mock_run.assert_called_with(["tree", str(update_dir)], check=False) + out, _ = capsys.readouterr() + assert "Affected Orgs and Environments" in out + assert "Org: org1, Envs: ['dev']" in out + assert "[Dry Run]" in out + + +def test_process_files_confirm_env(tmp_path): + update_dir = tmp_path / "update" / "resources" + update_dir.mkdir(parents=True) + (update_dir / "org1" / "env" / "prod").mkdir(parents=True) + (update_dir / "org1" / "env" / "prod" / "test.json").write_text("{}") + + with patch("subprocess.run") as mock_run: + process_files(str(tmp_path), "resources", confirm=True) + # Should call mvn install for prod env + mock_run.assert_called() + args, kwargs = mock_run.call_args + cmd = args[0] + assert "mvn" in cmd + assert "-Porg1" in cmd + assert "-Dapigee.env=prod" in cmd + assert "-Dapigee.org=org1" in cmd + + +def test_process_files_confirm_no_env(tmp_path): + update_dir = tmp_path / "update" / "resources" + update_dir.mkdir(parents=True) + (update_dir / "org1" / "org").mkdir(parents=True) + (update_dir / "org1" / "org" / "products.json").write_text("{}") + + with patch("subprocess.run") as mock_run: + process_files(str(tmp_path), "resources", confirm=True) + # Should call mvn install for org level + mock_run.assert_called() + args, kwargs = mock_run.call_args + cmd = args[0] + assert "mvn" in cmd + assert "-Porg1" in cmd + assert "-Dapigee.org=org1" in cmd + assert not any("-Dapigee.env=" in arg for arg in cmd) + + +def test_process_files_mvn_fail(tmp_path): + update_dir = tmp_path / "update" / "resources" + update_dir.mkdir(parents=True) + (update_dir / "org1" / "org").mkdir(parents=True) + (update_dir / "org1" / "org" / "products.json").write_text("{}") + + with patch("subprocess.run") as mock_run: + mock_run.side_effect = subprocess.CalledProcessError(1, ["mvn"]) + with pytest.raises(SystemExit) as e: + process_files(str(tmp_path), "resources", confirm=True) + assert e.value.code == 1 + + +def test_process_files_no_files(tmp_path, capsys): + (tmp_path / "update").mkdir() + process_files(str(tmp_path), "resources", confirm=False) + out, _ = capsys.readouterr() + assert "No files found in" in out + assert "(subfolder empty)." in out + + +def test_process_files_empty_action_dir(tmp_path, capsys): + (tmp_path / "update" / "resources").mkdir(parents=True) + process_files(str(tmp_path), "resources", confirm=False) + out, _ = capsys.readouterr() + assert "No files to process for update." in out + + +def test_process_files_mvn_fail_env(tmp_path): + update_dir = tmp_path / "update" / "resources" + update_dir.mkdir(parents=True) + (update_dir / "org1" / "env" / "dev").mkdir(parents=True) + (update_dir / "org1" / "env" / "dev" / "test.json").write_text("{}") + + with patch("subprocess.run") as mock_run: + mock_run.side_effect = subprocess.CalledProcessError(1, ["mvn"]) + with pytest.raises(SystemExit) as e: + process_files(str(tmp_path), "resources", confirm=True) + assert e.value.code == 1 + + +def test_process_files_tree_not_found(tmp_path, capsys): + update_dir = tmp_path / "update" / "resources" + update_dir.mkdir(parents=True) + (update_dir / "org1" / "org").mkdir(parents=True) + (update_dir / "org1" / "org" / "products.json").write_text("{}") + + with patch("subprocess.run") as mock_run: + mock_run.side_effect = FileNotFoundError + process_files(str(tmp_path), "resources", confirm=False) + out, _ = capsys.readouterr() + # Verify the fallback printing when tree is not found + assert "products.json" in out + + +def test_process_files_edge_cases(tmp_path, capsys): + # 1. No orgs/environments found + (tmp_path / "update" / "resources").mkdir(parents=True) + (tmp_path / "update" / "resources" / "invalid").write_text("{}") + process_files(str(tmp_path), "resources", confirm=False) + out, _ = capsys.readouterr() + assert "No orgs/environments found." in out + + # 2. empty org_env_map coverage + with patch("os.walk") as mock_walk: + mock_walk.return_value = [ + (str(tmp_path / "update" / "resources"), [], ["file.json"]) + ] + process_files(str(tmp_path), "resources", confirm=False) + out2, _ = capsys.readouterr() + assert "No orgs/environments found." in out2 + + +def test_process_files_auth_args(tmp_path): + update_dir = tmp_path / "update" / "resources" + update_dir.mkdir(parents=True) + (update_dir / "org1" / "org").mkdir(parents=True) + (update_dir / "org1" / "org" / "products.json").write_text("{}") + + with patch("subprocess.run") as mock_run: + process_files( + str(tmp_path), + "resources", + confirm=True, + bearer="mytoken", + sa_path="mysa.json", + ) + mock_run.assert_called() + args, _ = mock_run.call_args + cmd = args[0] + assert "-Dapigee.bearer=mytoken" in cmd + assert "-Dapigee.serviceaccount.file=mysa.json" in cmd + + +def test_process_files_auth_args_partial(tmp_path): + update_dir = tmp_path / "update" / "resources" + update_dir.mkdir(parents=True) + (update_dir / "org1" / "env" / "dev").mkdir(parents=True) + (update_dir / "org1" / "env" / "dev" / "test.json").write_text("{}") + with patch("subprocess.run") as mock_run: + process_files( + str(tmp_path), "resources", confirm=True, bearer="mytoken" + ) + assert "-Dapigee.bearer=mytoken" in mock_run.call_args[0][0] + assert not any( + "-Dapigee.serviceaccount.file=" in x + for x in mock_run.call_args[0][0] + ) diff --git a/tools/apigee-config-diff/tests/test_single_commit_integration.py b/tools/apigee-config-diff/tests/test_single_commit_integration.py new file mode 100644 index 00000000..26c869fc --- /dev/null +++ b/tools/apigee-config-diff/tests/test_single_commit_integration.py @@ -0,0 +1,70 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import patch, MagicMock +from apigee_config_diff.main import main +import os +import json + + +@patch( + "sys.argv", + [ + "apigee-config-diff", + "--commit-before", + "HEAD~1", + "--current-commit", + "HEAD", + "--folder", + "resources/", + "--output", + "./tmp/apigee-single", + ], +) +@patch("apigee_config_diff.diff.check.GitClient.list_files") +@patch("apigee_config_diff.diff.check.GitClient.read_file_contents") +@patch("subprocess.run") +def test_single_commit_integration( + mock_subprocess_run, mock_read_git_contents, mock_git_ls_files, tmp_path +): + # 1. Mock 'git rev-parse --verify HEAD~1' to FAIL (single commit repo) + mock_subprocess_run.return_value.returncode = 1 + + # 2. Mock 'git ls-files' to return some files + mock_ls_files_result = MagicMock() + mock_ls_files_result.stdout = ( + "resources/my-org/org/apiProducts.json\npom.xml\n" + ) + mock_git_ls_files.return_value = mock_ls_files_result + + # 3. Mock file contents + mock_read_git_contents.return_value = '{"name": "my-product"}' + + # 4. Run main + main() + + # Verify: pom.xml should be ignored, + # resources/my-org/org/apiProducts.json should be in update/ + update_file = ( + "./tmp/apigee-single/update/resources/my-org/org/apiProducts.json" + ) + assert os.path.exists(update_file) + + # Verify: pom.xml should NOT be in the update folder + pom_file = "./tmp/apigee-single/update/pom.xml" + assert not os.path.exists(pom_file) + + with open(update_file) as f: + content = json.load(f) + assert content["name"] == "my-product" diff --git a/tools/apigee-config-diff/tests/test_util.py b/tools/apigee-config-diff/tests/test_util.py new file mode 100644 index 00000000..e6e09fb5 --- /dev/null +++ b/tools/apigee-config-diff/tests/test_util.py @@ -0,0 +1,149 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import os +from unittest.mock import patch +from apigee_config_diff.diff.util import ( + GitClient, + create_folder, + find_resource_type, + write_to_file, + run_command_or_exit, + merge, +) + + +def test_resolve_commits_normal(): + with patch("subprocess.run") as mock_run: + mock_run.return_value.returncode = 0 + assert GitClient.resolve_commits("abc", "def") == ("abc", "def") + + +def test_resolve_commits_zeros(): + # Verifies that zeros always return "" for previous commit + assert GitClient.resolve_commits("0", "def") == ("", "def") + assert GitClient.resolve_commits("0000000", "def") == ("", "def") + + +def test_resolve_commits_single_commit(): + # Simulates single commit scenario where HEAD~1 does not exist + with patch("subprocess.run") as mock_run: + mock_run.return_value.returncode = 1 + assert GitClient.resolve_commits("HEAD~1", "HEAD") == ("", "HEAD") + + +def test_resolve_commits_git_not_found(): + with patch("subprocess.run") as mock_run: + mock_run.side_effect = FileNotFoundError + with pytest.raises(SystemExit) as e: + GitClient.resolve_commits("HEAD", "def") + assert e.value.code == 1 + + +@patch("apigee_config_diff.diff.util.run_command_or_exit") +def test_read_file_contents(mock_run): + mock_run.return_value.stdout = "content" + assert GitClient.read_file_contents("hash", "path") == "content" + + +@patch("apigee_config_diff.diff.util.run_command_or_exit") +def test_diff_hashes(mock_run): + GitClient.diff_hashes("a", "b") + mock_run.assert_called_once_with( + ["git", "diff", "--name-status", "a", "b"], capture_output=True + ) + + +def test_create_folder(tmp_path): + folder = tmp_path / "test" + folder.mkdir() + (folder / "file.txt").write_text("hello") + + new_folder = create_folder(str(folder)) + assert os.path.exists(new_folder) + assert len(os.listdir(new_folder)) == 0 + + +def test_find_resource_type(): + types = ["kvms", "targetServers"] + assert find_resource_type("kvms.json", types) == "kvms" + assert find_resource_type("unknown.json", types) is None + + +def test_write_to_file(tmp_path): + f_path = tmp_path / "sub" / "test.json" + content = {"a": 1} + write_to_file(str(f_path), content) + assert f_path.exists() + import json + + with open(f_path) as f: + assert json.load(f) == content + + +def test_run_command_or_exit_success(): + # Use a real shell command that will succeed + res = run_command_or_exit(["echo", "hello"], capture_output=True) + assert res.stdout.strip() == "hello" + assert res.returncode == 0 + + +def test_run_command_or_exit_not_found(): + # Command doesn't exist + with pytest.raises(SystemExit) as e: + run_command_or_exit(["nonexistent_command_12345"]) + assert e.value.code == 1 + + +def test_run_command_or_exit_fail(): + # Use a python command that prints to stdout, stderr, and then exits 1 + with pytest.raises(SystemExit) as e: + run_command_or_exit( + [ + "python3", + "-c", + "import sys; print('out'); " + "print('err', file=sys.stderr); sys.exit(1)", + ], + capture_output=True, + ) + assert e.value.code == 1 + + +def test_merge_primitives(): + assert merge(1, 2) == 2 + assert merge(1, None) == 1 + assert merge(None, 2) == 2 + + +def test_merge_lists(): + assert merge([1], [2]) == [1, 2] + + +def test_merge_dicts(): + a = {"k1": [1], "k2": {"s1": 1}} + b = {"k1": [2], "k2": {"s1": 2, "s2": 3}} + expected = {"k1": [1, 2], "k2": {"s1": 2, "s2": 3}} + assert merge(a, b) == expected + + +@patch("apigee_config_diff.diff.util.run_command_or_exit") +def test_list_files(mock_run_command_or_exit): + mock_run_command_or_exit.return_value = "files" + result = GitClient.list_files() + mock_run_command_or_exit.assert_called_once_with( + ["git", "ls-files"], capture_output=True + ) + assert result == "files"