direct: Fix permissions state path to match input config schema

NEXT_CHANGELOG.md

@@ -5,6 +5,7 @@
 ### CLI
 
 ### Bundles
+* direct: Fix permissions state path to match input config schema ([#4703](https://github.com/databricks/cli/pull/4703))
 
 ### Dependency updates
 

acceptance/acceptance_test.go

@@ -209,6 +209,12 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 	t.Setenv("CLI", execPath)
 	repls.SetPath(execPath, "[CLI]")
 
+	if !inprocessMode {
+		cli293Path := DownloadCLI(t, buildDir, "0.293.0")
+		t.Setenv("CLI_293", cli293Path)
+		repls.SetPath(cli293Path, "[CLI_293]")
+	}
+
 	paths := []string{
 		// Make helper scripts available
 		filepath.Join(cwd, "bin"),

acceptance/bundle/apps/job_permissions/test.toml

@@ -1,11 +1,5 @@
-# Direct engine error: cannot plan resources.jobs.my_job.permissions: cannot update
-# [0].service_principal_name: failed to navigate to parent [0]: [0]: cannot index struct.
-# This is a bug in structaccess.Set() where it fails to index into a struct when
-# setting permissions with service_principal_name.
-# See https://github.com/databricks/cli/pull/4644
-Badness = "Direct engine fails to plan permissions with service_principal_name on jobs"
 Cloud = true
 RecordRequests = false
 
 [EnvMatrix]
-DATABRICKS_BUNDLE_ENGINE = ["terraform"]
+DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"]

acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json

@@ -1,5 +1,5 @@
 {
- "state_version": 1,
+ "state_version": 2,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 2,

acceptance/bundle/invariant/configs/job_cross_resource_ref.yml.tmpl

@@ -0,0 +1,27 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  jobs:
+    # job_src defines permissions and a tag value used as references by other resources
+    job_src:
+      name: test-job-src-$UNIQUE_NAME
+      tags:
+        perm_group: users
+      permissions:
+        - level: CAN_VIEW
+          group_name: users
+
+    # job_perm_ref uses permission fields from job_src as its permission values
+    job_perm_ref:
+      name: test-job-perm-ref-$UNIQUE_NAME
+      permissions:
+        - level: ${resources.jobs.job_src.permissions[0].level}
+          group_name: ${resources.jobs.job_src.permissions[0].group_name}
+
+    # job_tag_ref uses a job tag from job_src as a permission group_name
+    job_tag_ref:
+      name: test-job-tag-ref-$UNIQUE_NAME
+      permissions:
+        - level: CAN_VIEW
+          group_name: ${resources.jobs.job_src.tags.perm_group}

acceptance/bundle/invariant/configs/job_permission_ref.yml.tmpl

@@ -0,0 +1,21 @@
+bundle:
+  name: test-bundle-$UNIQUE_NAME
+
+resources:
+  jobs:
+    job_b:
+      name: test-job-b-$UNIQUE_NAME
+      permissions:
+        - level: CAN_VIEW
+          group_name: users
+        - level: CAN_MANAGE
+          group_name: admins
+
+    job_a:
+      name: test-job-a-$UNIQUE_NAME
+      permissions:
+        # Reference level and group_name from job_b by index
+        - level: ${resources.jobs.job_b.permissions[0].level}
+          group_name: ${resources.jobs.job_b.permissions[0].group_name}
+        - level: ${resources.jobs.job_b.permissions[1].level}
+          group_name: ${resources.jobs.job_b.permissions[1].group_name}

acceptance/bundle/invariant/continue_293/output.txt

@@ -0,0 +1 @@
+INPUT_CONFIG_OK

acceptance/bundle/invariant/continue_293/script

@@ -0,0 +1,40 @@
+# Invariant to test: current CLI can deploy on top of state produced by v0.293.0
+
+cp -r "$TESTDIR/../data/." . &> LOG.cp
+
+INIT_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-init.sh"
+if [ -f "$INIT_SCRIPT" ]; then
+    source "$INIT_SCRIPT" &> LOG.init
+fi
+
+envsubst < "$TESTDIR/../configs/$INPUT_CONFIG" > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve &> LOG.destroy
+    cat LOG.destroy | contains.py '!panic' '!internal error' > /dev/null
+
+    CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh"
+    if [ -f "$CLEANUP_SCRIPT" ]; then
+        source "$CLEANUP_SCRIPT" &> LOG.cleanup
+    fi
+}
+
+trap cleanup EXIT
+
+# Deploy with old CLI to produce v0.293.0 state
+trace $CLI_293 bundle deploy &> LOG.deploy.293
+cat LOG.deploy.293 | contains.py '!panic' '!internal error' > /dev/null
+
+echo INPUT_CONFIG_OK
+
+# Deploy with current CLI on top of old state
+trace $CLI bundle deploy &> LOG.deploy
+cat LOG.deploy | contains.py '!panic' '!internal error' > /dev/null
+
+# Verify no drift after current CLI deploy
+$CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err
+cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null
+verify_no_drift.py LOG.planjson
+
+$CLI bundle plan 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan
+cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null

acceptance/bundle/invariant/continue_293/test.toml

@@ -0,0 +1,5 @@
+Cloud = false
+
+# Cross-resource permission references don't work in terraform mode.
+EnvMatrixExclude.no_permission_ref = ["INPUT_CONFIG=job_permission_ref.yml.tmpl"]
+EnvMatrixExclude.no_cross_resource_ref = ["INPUT_CONFIG=job_cross_resource_ref.yml.tmpl"]

acceptance/bundle/invariant/migrate/script

@@ -38,6 +38,6 @@ trace $CLI bundle deployment migrate &> LOG.migrate
 
 cat LOG.migrate | contains.py '!panic:' '!internal error' > /dev/null
 
-$CLI bundle plan -o json &> plan.json
-cat plan.json | contains.py '!panic:' '!internal error' > /dev/null
+$CLI bundle plan -o json > plan.json 2>plan.json.err
+cat plan.json.err | contains.py '!panic:' '!internal error' > /dev/null
 verify_no_drift.py plan.json

acceptance/bundle/invariant/migrate/test.toml

@@ -4,3 +4,12 @@ EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmp
 
 # Unexpected action='create' for resources.secret_scopes.foo.permissions
 EnvMatrixExclude.no_secret_scope = ["INPUT_CONFIG=secret_scope.yml.tmpl"]
+
+# Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level})
+# don't work in terraform mode: the terraform interpolator converts the path to
+# ${databricks_job.job_b.permissions[0].level}, but Terraform's databricks_job resource
+# does not expose permissions as output attributes (permissions are a separate
+# databricks_permissions resource in terraform), so the literal unresolved string
+# ends up as the permission level value.
+EnvMatrixExclude.no_permission_ref = ["INPUT_CONFIG=job_permission_ref.yml.tmpl"]
+EnvMatrixExclude.no_cross_resource_ref = ["INPUT_CONFIG=job_cross_resource_ref.yml.tmpl"]

acceptance/bundle/invariant/no_drift/script

@@ -41,8 +41,9 @@ echo INPUT_CONFIG_OK
 
 # Check both text and JSON plan for no changes
 # Note, expect that there maybe more than one resource unchanged
-$CLI bundle plan -o json &> LOG.planjson
-cat LOG.planjson | contains.py '!panic' '!internal error' > /dev/null
+$CLI bundle plan -o json > LOG.planjson 2>LOG.planjson.err
+cat LOG.planjson.err | contains.py '!panic' '!internal error' > /dev/null
 verify_no_drift.py LOG.planjson
 
-$CLI bundle plan | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan
+$CLI bundle plan 2>LOG.plan.err | contains.py '!panic' '!internal error' 'Plan: 0 to add, 0 to change, 0 to delete' > LOG.plan
+cat LOG.plan.err | contains.py '!panic' '!internal error' > /dev/null

acceptance/bundle/invariant/test.toml

@@ -10,6 +10,7 @@ Ignore = [
     "plan.json",
     "*.py",
     "*.json",
+    "*.err",
     "app",
 ]
 
@@ -32,6 +33,8 @@ EnvMatrix.INPUT_CONFIG = [
     "job.yml.tmpl",
     "job_pydabs_10_tasks.yml.tmpl",
     "job_pydabs_1000_tasks.yml.tmpl",
+    "job_cross_resource_ref.yml.tmpl",
+    "job_permission_ref.yml.tmpl",
     "job_with_task.yml.tmpl",
     "model.yml.tmpl",
     "model_serving_endpoint.yml.tmpl",

acceptance/bundle/migrate/basic/out.new_state.json

@@ -1,5 +1,5 @@
 {
- "state_version": 1,
+ "state_version": 2,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 6,

acceptance/bundle/migrate/dashboards/out.new_state.json

@@ -1,5 +1,5 @@
 {
- "state_version": 1,
+ "state_version": 2,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 3,

acceptance/bundle/migrate/default-python/out.state_after_migration.json

@@ -1,5 +1,5 @@
 {
- "state_version": 1,
+ "state_version": 2,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 5,

acceptance/bundle/migrate/grants/out.new_state.json

@@ -1,5 +1,5 @@
 {
- "state_version": 1,
+ "state_version": 2,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 9,

acceptance/bundle/migrate/permissions/out.new_state.json

@@ -1,5 +1,5 @@
 {
- "state_version": 1,
+ "state_version": 2,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 7,
@@ -32,13 +32,13 @@
    "__id__": "/jobs/[NUMID]",
    "state": {
     "object_id": "/jobs/[NUMID]",
-    "permissions": [
+    "_": [
      {
-      "permission_level": "CAN_VIEW",
+      "level": "CAN_VIEW",
       "user_name": "viewer@databricks.com"
      },
      {
-      "permission_level": "IS_OWNER",
+      "level": "IS_OWNER",
       "user_name": "[USERNAME]"
      }
     ]
@@ -73,13 +73,13 @@
    "__id__": "/pipelines/[UUID]",
    "state": {
     "object_id": "/pipelines/[UUID]",
-    "permissions": [
+    "_": [
      {
-      "permission_level": "CAN_MANAGE",
+      "level": "CAN_MANAGE",
       "user_name": "manager@databricks.com"
      },
      {
-      "permission_level": "IS_OWNER",
+      "level": "IS_OWNER",
       "user_name": "[USERNAME]"
      }
     ]

acceptance/bundle/migrate/runas/out.new_state.json

@@ -1,5 +1,5 @@
 {
- "state_version": 1,
+ "state_version": 2,
  "cli_version": "[DEV_VERSION]",
  "lineage": "[UUID]",
  "serial": 5,
@@ -33,13 +33,13 @@
    "__id__": "/pipelines/[UUID]",
    "state": {
     "object_id": "/pipelines/[UUID]",
-    "permissions": [
+    "_": [
      {
-      "group_name": "users",
-      "permission_level": "CAN_VIEW"
+      "level": "CAN_VIEW",
+      "group_name": "users"
      },
      {
-      "permission_level": "IS_OWNER",
+      "level": "IS_OWNER",
       "user_name": "[USERNAME]"
      }
     ]

acceptance/bundle/migrate/runas/out.plan.json

@@ -54,13 +54,13 @@
       "action": "skip",
       "remote_state": {
         "object_id": "/pipelines/[UUID]",
-        "permissions": [
+        "_": [
           {
-            "group_name": "users",
-            "permission_level": "CAN_VIEW"
+            "level": "CAN_VIEW",
+            "group_name": "users"
           },
           {
-            "permission_level": "IS_OWNER",
+            "level": "IS_OWNER",
             "user_name": "[USERNAME]"
           }
         ]