From e22b35c1609abef251af758bfceb2464e5ccfa9b Mon Sep 17 00:00:00 2001 From: Kaiohz Date: Mon, 6 Apr 2026 18:40:50 +0200 Subject: [PATCH] feat(composables): add raganything deployment with minio and postgres infrastructure - Add MinIO deployment, service, ingress, and bucket init job - Add RagAnything API deployment with configmap and ingress - Add PostgreSQL deployment with persistence - Add composable-agents deployment - Configure external secrets for OpenBao integration - Set up persistent volumes on NFS storage Infrastructure dependencies: - MinIO: Object storage for RAG documents - PostgreSQL (pgvector + Apache AGE): Knowledge graph storage - RagAnything: FastAPI service exposing REST and MCP endpoints Services exposed: - raganything.soludev.tech - RagAnything API + MCP - composables-minio.soludev.tech - MinIO Console - composables-api-minio.soludev.tech - MinIO API --- .../composable-agents/configmap.yaml | 33 ++++++++++ .../composable-agents/deployment.yaml | 49 ++++++++++++++ .../composable-agents/external-secret.yaml | 16 +++++ .../composable-agents/ingress.yaml | 20 ++++++ .../composable-agents/service.yaml | 13 ++++ dev/composables/minio/bucket-init-job.yaml | 26 ++++++++ dev/composables/minio/deployment.yaml | 64 +++++++++++++++++++ dev/composables/minio/external-secret.yaml | 16 +++++ dev/composables/minio/ingress.yaml | 30 +++++++++ dev/composables/minio/persistent-volume.yaml | 20 ++++++ dev/composables/minio/service.yaml | 18 ++++++ dev/composables/minio/volume-claim.yaml | 12 ++++ dev/composables/postgres/deployment.yaml | 49 ++++++++++++++ dev/composables/postgres/external-secret.yaml | 16 +++++ .../postgres/persistent-volume.yaml | 20 ++++++ dev/composables/postgres/service.yaml | 13 ++++ dev/composables/postgres/volume-claim.yaml | 12 ++++ dev/composables/raganything/configmap.yaml | 31 +++++++++ dev/composables/raganything/deployment.yaml | 51 +++++++++++++++ .../raganything/external-secret.yaml | 16 +++++ dev/composables/raganything/ingress.yaml | 20 ++++++ dev/composables/raganything/service.yaml | 13 ++++ 22 files changed, 558 insertions(+) create mode 100644 dev/composables/composable-agents/configmap.yaml create mode 100644 dev/composables/composable-agents/deployment.yaml create mode 100644 dev/composables/composable-agents/external-secret.yaml create mode 100644 dev/composables/composable-agents/ingress.yaml create mode 100644 dev/composables/composable-agents/service.yaml create mode 100644 dev/composables/minio/bucket-init-job.yaml create mode 100644 dev/composables/minio/deployment.yaml create mode 100644 dev/composables/minio/external-secret.yaml create mode 100644 dev/composables/minio/ingress.yaml create mode 100644 dev/composables/minio/persistent-volume.yaml create mode 100644 dev/composables/minio/service.yaml create mode 100644 dev/composables/minio/volume-claim.yaml create mode 100644 dev/composables/postgres/deployment.yaml create mode 100644 dev/composables/postgres/external-secret.yaml create mode 100644 dev/composables/postgres/persistent-volume.yaml create mode 100644 dev/composables/postgres/service.yaml create mode 100644 dev/composables/postgres/volume-claim.yaml create mode 100644 dev/composables/raganything/configmap.yaml create mode 100644 dev/composables/raganything/deployment.yaml create mode 100644 dev/composables/raganything/external-secret.yaml create mode 100644 dev/composables/raganything/ingress.yaml create mode 100644 dev/composables/raganything/service.yaml diff --git a/dev/composables/composable-agents/configmap.yaml b/dev/composables/composable-agents/configmap.yaml new file mode 100644 index 0000000..e86e5d9 --- /dev/null +++ b/dev/composables/composable-agents/configmap.yaml @@ -0,0 +1,33 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: composable-agents-config + namespace: composables +data: + # === OpenAI-compatible endpoint (OpenRouter, LiteLLM, etc.) === + OPENAI_BASE_URL: "https://openrouter.ai/api/v1" + # === Agent configuration === + AGENTS_DIR: "./agents" + HOST: "0.0.0.0" + PORT: "8000" + # === MinIO (object storage) === + MINIO_ENDPOINT: "bricks-minio:9000" + MINIO_ACCESS_KEY: "minioadmin" + MINIO_BUCKET: "composable-agents" + MINIO_SECURE: "false" + # === PostgreSQL (metadata) === + POSTGRES_HOST: "bricks-db" + POSTGRES_PORT: "5432" + POSTGRES_USER: "raganything" + POSTGRES_DATABASE: "raganything" + # === Tracing (optional) === + TRACING_PROVIDER: "none" + TRACING_ENABLED: "false" + TRACING_PROJECT_NAME: "composable-agents" + LANGFUSE_HOST: "https://cloud.langfuse.com" + LANGFUSE_PUBLIC_KEY: "" + LANGFUSE_SECRET_KEY: "" + PHOENIX_COLLECTOR_ENDPOINT: "http://bricks-phoenix:6006" + PHOENIX_API_KEY: "" + LANGCHAIN_API_KEY: "" + LANGCHAIN_PROJECT: "composable-agents" \ No newline at end of file diff --git a/dev/composables/composable-agents/deployment.yaml b/dev/composables/composable-agents/deployment.yaml new file mode 100644 index 0000000..a34a101 --- /dev/null +++ b/dev/composables/composable-agents/deployment.yaml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: composable-agents + namespace: composables + labels: + app: composable-agents +spec: + replicas: 1 + selector: + matchLabels: + app: composable-agents + template: + metadata: + labels: + app: composable-agents + spec: + tolerations: + - key: "klipperlb-exclude" + value: "true" + effect: "NoSchedule" + operator: "Equal" + containers: + - name: composable-agents + image: kaiohz/composable-agents:latest + ports: + - containerPort: 8000 + envFrom: + - secretRef: + name: composable-agents-secret + livenessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 90 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /health + port: 8000 + initialDelaySeconds: 60 + periodSeconds: 5 + resources: + requests: + memory: "256Mi" + cpu: "200m" + limits: + memory: "512Mi" + cpu: "500m" diff --git a/dev/composables/composable-agents/external-secret.yaml b/dev/composables/composable-agents/external-secret.yaml new file mode 100644 index 0000000..9fc8f31 --- /dev/null +++ b/dev/composables/composable-agents/external-secret.yaml @@ -0,0 +1,16 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: composable-agents-external-secret + namespace: composables +spec: + refreshInterval: 60s + secretStoreRef: + name: openbao-backend + kind: ClusterSecretStore + target: + name: composable-agents-secret + creationPolicy: Owner + dataFrom: + - extract: + key: composables/composable-agents diff --git a/dev/composables/composable-agents/ingress.yaml b/dev/composables/composable-agents/ingress.yaml new file mode 100644 index 0000000..27af49d --- /dev/null +++ b/dev/composables/composable-agents/ingress.yaml @@ -0,0 +1,20 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: composable-agents-ingress + namespace: composables + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web,websecure + traefik.ingress.kubernetes.io/redirect-scheme: https +spec: + rules: + - host: composable-agents.soludev.tech + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: composable-agents + port: + number: 8000 diff --git a/dev/composables/composable-agents/service.yaml b/dev/composables/composable-agents/service.yaml new file mode 100644 index 0000000..b9ecd26 --- /dev/null +++ b/dev/composables/composable-agents/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: composable-agents + namespace: composables +spec: + selector: + app: composable-agents + ports: + - protocol: TCP + port: 8000 + targetPort: 8000 + type: ClusterIP diff --git a/dev/composables/minio/bucket-init-job.yaml b/dev/composables/minio/bucket-init-job.yaml new file mode 100644 index 0000000..8eb3495 --- /dev/null +++ b/dev/composables/minio/bucket-init-job.yaml @@ -0,0 +1,26 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-bucket-init + namespace: composables +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: mc + image: minio/mc:latest + envFrom: + - secretRef: + name: minio-secret + env: + - name: MINIO_ENDPOINT + value: "minio:9000" + command: + - /bin/sh + - -c + - | + mc alias set local http://${MINIO_ENDPOINT} ${MINIO_ROOT_USER} ${MINIO_ROOT_PASSWORD} + mc mb --ignore-existing local/raganything + mc mb --ignore-existing local/composable-agents + echo "Buckets created successfully" \ No newline at end of file diff --git a/dev/composables/minio/deployment.yaml b/dev/composables/minio/deployment.yaml new file mode 100644 index 0000000..ca05454 --- /dev/null +++ b/dev/composables/minio/deployment.yaml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: composables + labels: + app: minio +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + tolerations: + - key: "klipperlb-exclude" + value: "true" + effect: "NoSchedule" + operator: "Equal" + containers: + - name: minio + image: minio/minio:latest + args: + - server + - /data + - --console-address + - ":9001" + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + envFrom: + - secretRef: + name: minio-secret + volumeMounts: + - name: minio-data + mountPath: /data + livenessProbe: + httpGet: + path: /minio/health/live + port: 9000 + initialDelaySeconds: 30 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /minio/health/ready + port: 9000 + initialDelaySeconds: 10 + periodSeconds: 5 + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "500m" + volumes: + - name: minio-data + persistentVolumeClaim: + claimName: minio \ No newline at end of file diff --git a/dev/composables/minio/external-secret.yaml b/dev/composables/minio/external-secret.yaml new file mode 100644 index 0000000..0691320 --- /dev/null +++ b/dev/composables/minio/external-secret.yaml @@ -0,0 +1,16 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: minio-external-secret + namespace: composables +spec: + refreshInterval: 60s + secretStoreRef: + name: openbao-backend + kind: ClusterSecretStore + target: + name: minio-secret + creationPolicy: Owner + dataFrom: + - extract: + key: composables/minio \ No newline at end of file diff --git a/dev/composables/minio/ingress.yaml b/dev/composables/minio/ingress.yaml new file mode 100644 index 0000000..7d0d37f --- /dev/null +++ b/dev/composables/minio/ingress.yaml @@ -0,0 +1,30 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: minio-ingress + namespace: composables + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web,websecure + traefik.ingress.kubernetes.io/redirect-scheme: https +spec: + rules: + - host: composables-minio.soludev.tech + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: minio + port: + number: 9001 + - host: composables-api-minio.soludev.tech + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: minio + port: + number: 9000 diff --git a/dev/composables/minio/persistent-volume.yaml b/dev/composables/minio/persistent-volume.yaml new file mode 100644 index 0000000..78c4efa --- /dev/null +++ b/dev/composables/minio/persistent-volume.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nfs-composables-minio + namespace: composables +spec: + capacity: + storage: 5Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: nfs-composables-minio + nfs: + server: 100.64.0.6 + path: /Volumes/NFSStorage/composables/minio + mountOptions: + - nfsvers=3 + - hard + - timeo=600 + - retrans=2 diff --git a/dev/composables/minio/service.yaml b/dev/composables/minio/service.yaml new file mode 100644 index 0000000..950174a --- /dev/null +++ b/dev/composables/minio/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: composables +spec: + selector: + app: minio + ports: + - name: api + protocol: TCP + port: 9000 + targetPort: 9000 + - name: console + protocol: TCP + port: 9001 + targetPort: 9001 + type: ClusterIP \ No newline at end of file diff --git a/dev/composables/minio/volume-claim.yaml b/dev/composables/minio/volume-claim.yaml new file mode 100644 index 0000000..1c979e3 --- /dev/null +++ b/dev/composables/minio/volume-claim.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio + namespace: composables +spec: + accessModes: + - ReadWriteMany + storageClassName: nfs-composables-minio + resources: + requests: + storage: 5Gi diff --git a/dev/composables/postgres/deployment.yaml b/dev/composables/postgres/deployment.yaml new file mode 100644 index 0000000..3c1ec2e --- /dev/null +++ b/dev/composables/postgres/deployment.yaml @@ -0,0 +1,49 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: postgres + namespace: composables + labels: + app: postgres +spec: + replicas: 1 + selector: + matchLabels: + app: postgres + template: + metadata: + labels: + app: postgres + spec: + tolerations: + - key: "klipperlb-exclude" + value: "true" + effect: "NoSchedule" + operator: "Equal" + volumes: + - name: postgres-data + persistentVolumeClaim: + claimName: composables-postgres-pvc + containers: + - name: postgres + image: postgres:17-alpine + ports: + - containerPort: 5432 + envFrom: + - secretRef: + name: postgres-secret + volumeMounts: + - name: postgres-data + mountPath: /var/lib/postgresql/data + readinessProbe: + exec: + command: ["pg_isready", "-U", "composable_agents", "-d", "composable_agents"] + initialDelaySeconds: 5 + periodSeconds: 5 + resources: + requests: + memory: "256Mi" + cpu: "300m" + limits: + memory: "512Mi" + cpu: "500m" diff --git a/dev/composables/postgres/external-secret.yaml b/dev/composables/postgres/external-secret.yaml new file mode 100644 index 0000000..42a56b4 --- /dev/null +++ b/dev/composables/postgres/external-secret.yaml @@ -0,0 +1,16 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: postgres-external-secret + namespace: composables +spec: + refreshInterval: 60s + secretStoreRef: + name: openbao-backend + kind: ClusterSecretStore + target: + name: postgres-secret + creationPolicy: Owner + dataFrom: + - extract: + key: composables/postgres diff --git a/dev/composables/postgres/persistent-volume.yaml b/dev/composables/postgres/persistent-volume.yaml new file mode 100644 index 0000000..e0d8ac2 --- /dev/null +++ b/dev/composables/postgres/persistent-volume.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: nfs-composables-postgres + namespace: composables +spec: + capacity: + storage: 5Gi + accessModes: + - ReadWriteMany + persistentVolumeReclaimPolicy: Retain + storageClassName: nfs-composables-postgres + nfs: + server: 100.64.0.6 + path: /Volumes/NFSStorage/composables/postgres + mountOptions: + - nfsvers=3 + - hard + - timeo=600 + - retrans=2 diff --git a/dev/composables/postgres/service.yaml b/dev/composables/postgres/service.yaml new file mode 100644 index 0000000..a1572e2 --- /dev/null +++ b/dev/composables/postgres/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres + namespace: composables +spec: + selector: + app: postgres + ports: + - protocol: TCP + port: 5432 + targetPort: 5432 + type: ClusterIP diff --git a/dev/composables/postgres/volume-claim.yaml b/dev/composables/postgres/volume-claim.yaml new file mode 100644 index 0000000..ec726c5 --- /dev/null +++ b/dev/composables/postgres/volume-claim.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: composables-postgres-pvc + namespace: composables +spec: + accessModes: + - ReadWriteMany + storageClassName: nfs-composables-postgres + resources: + requests: + storage: 5Gi diff --git a/dev/composables/raganything/configmap.yaml b/dev/composables/raganything/configmap.yaml new file mode 100644 index 0000000..0d660f4 --- /dev/null +++ b/dev/composables/raganything/configmap.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: raganything-config + namespace: composables +data: + HOST: "0.0.0.0" + PORT: "8000" + MCP_TRANSPORT: "sse" + ALLOWED_ORIGINS: '["*"]' + POSTGRES_HOST: "postgres" + POSTGRES_PORT: "5432" + POSTGRES_DATABASE: "raganything" + POSTGRES_USER: "raganything" + MINIO_HOST: "minio:9000" + MINIO_BUCKET: "raganything" + MINIO_SECURE: "false" + RAG_STORAGE_TYPE: "postgres" + COSINE_THRESHOLD: "0.2" + MAX_CONCURRENT_FILES: "1" + MAX_WORKERS: "1" + ENABLE_IMAGE_PROCESSING: "true" + ENABLE_TABLE_PROCESSING: "true" + ENABLE_EQUATION_PROCESSING: "true" + CHAT_MODEL: "openai/gpt-4o-mini" + EMBEDDING_MODEL: "text-embedding-3-small" + EMBEDDING_DIM: "1536" + MAX_TOKEN_SIZE: "8192" + VISION_MODEL: "openai/gpt-4o" + OPEN_ROUTER_API_URL: "https://openrouter.ai/api/v1" + UVICORN_LOG_LEVEL: "critical" \ No newline at end of file diff --git a/dev/composables/raganything/deployment.yaml b/dev/composables/raganything/deployment.yaml new file mode 100644 index 0000000..eb927e8 --- /dev/null +++ b/dev/composables/raganything/deployment.yaml @@ -0,0 +1,51 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: raganything + namespace: composables + labels: + app: raganything +spec: + replicas: 1 + selector: + matchLabels: + app: raganything + template: + metadata: + labels: + app: raganything + spec: + tolerations: + - key: "klipperlb-exclude" + value: "true" + effect: "NoSchedule" + operator: "Equal" + containers: + - name: raganything + image: kaiohz/raganything:latest + ports: + - containerPort: 8000 + envFrom: + - configMapRef: + name: raganything-config + - secretRef: + name: raganything-secret + livenessProbe: + httpGet: + path: /api/v1/health + port: 8000 + initialDelaySeconds: 120 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /api/v1/health + port: 8000 + initialDelaySeconds: 90 + periodSeconds: 5 + resources: + requests: + memory: "512Mi" + cpu: "500m" + limits: + memory: "2Gi" + cpu: "2000m" \ No newline at end of file diff --git a/dev/composables/raganything/external-secret.yaml b/dev/composables/raganything/external-secret.yaml new file mode 100644 index 0000000..6874c17 --- /dev/null +++ b/dev/composables/raganything/external-secret.yaml @@ -0,0 +1,16 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: raganything-external-secret + namespace: composables +spec: + refreshInterval: 60s + secretStoreRef: + name: openbao-backend + kind: ClusterSecretStore + target: + name: raganything-secret + creationPolicy: Owner + dataFrom: + - extract: + key: composables/raganything \ No newline at end of file diff --git a/dev/composables/raganything/ingress.yaml b/dev/composables/raganything/ingress.yaml new file mode 100644 index 0000000..2d3a471 --- /dev/null +++ b/dev/composables/raganything/ingress.yaml @@ -0,0 +1,20 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: raganything-ingress + namespace: composables + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web,websecure + traefik.ingress.kubernetes.io/redirect-scheme: https +spec: + rules: + - host: raganything.soludev.tech + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: raganything + port: + number: 8000 \ No newline at end of file diff --git a/dev/composables/raganything/service.yaml b/dev/composables/raganything/service.yaml new file mode 100644 index 0000000..b9f0720 --- /dev/null +++ b/dev/composables/raganything/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: raganything + namespace: composables +spec: + selector: + app: raganything + ports: + - protocol: TCP + port: 8000 + targetPort: 8000 + type: ClusterIP \ No newline at end of file