Operations

# Forward all services
./scripts/port-forward.sh
 
# Infrastructure only
./scripts/port-forward-infra.sh

make setup-datalake-kind
make verify-bigdata-kind
make verify-bigdata-kind-readiness
make verify-bigdata-kind-smoke
make teardown-bigdata-kind

# Back up everything: PostgreSQL, Redis, Qdrant, Neo4j, MinIO, Consul, NATS, MQTT
./scripts/backup-cluster-data.sh
 
# Custom backup location
./scripts/backup-cluster-data.sh /path/to/backup
 
# Specify cluster name
KIND_CLUSTER=isa-cloud-staging ./scripts/backup-cluster-data.sh

# Using the NATS CLI (preferred)
nats stream list --json > streams.json
 
# Per-stream config export
for stream in $(nats stream list --json | jq -r '.[].config.name'); do
  nats stream info "$stream" --json > "nats/stream_${stream}.json"
  # Consumer configs for each stream
  for consumer in $(nats consumer list "$stream" --json | jq -r '.[].config.durable_name // empty'); do
    nats consumer info "$stream" "$consumer" --json > "nats/consumer_${stream}_${consumer}.json"
  done
done

kubectl exec -n isa-cloud-staging deploy/nats -- \
  tar czf /tmp/jetstream.tar.gz /data/jetstream
 
kubectl cp isa-cloud-staging/nats-0:/tmp/jetstream.tar.gz \
  ./nats-backup-$(date +%Y%m%d).tar.gz

# Atomic snapshot (captures KV + service catalog + ACLs)
consul snapshot save /tmp/consul-backup.snap
 
# Or via the HTTP API
curl -sf -o consul-snapshot.snap http://localhost:8500/v1/snapshot
 
# Fallback: KV-only export as JSON
consul kv export > consul-kv-$(date +%Y%m%d).json

kubectl cp isa-cloud-staging/<mosquitto-pod>:/mosquitto/data/mosquitto.db \
  ./mqtt-backup-$(date +%Y%m%d)/mosquitto.db

# Key CronJob configuration
spec:
  schedule: "0 2 * * *"          # Daily at 2 AM UTC
  concurrencyPolicy: Forbid      # Never run two backups at once
  successfulJobsHistoryLimit: 7
  failedJobsHistoryLimit: 3

# Full restore (latest)
kubectl exec -it postgresql-backup-pod -- bash /scripts/restore.sh
 
# Restore specific timestamp
kubectl exec -it postgresql-backup-pod -- bash /scripts/restore.sh pgdump_20260309_020000
 
# Restore single database (uses pg_restore with parallel jobs)
kubectl exec -it postgresql-backup-pod -- bash /scripts/restore.sh --db isa_production

# PostgreSQL
kubectl exec -it postgres-0 -n isa-cloud-staging -- \
  pg_dump -U postgres isa_db | gzip > backup.sql.gz
 
# Redis
kubectl exec -it redis-master-0 -n isa-cloud-staging -- \
  redis-cli BGSAVE
 
# MinIO
mc mirror minio/bucket ./backup/bucket

./scripts/restore-cluster-data.sh backups/cluster-backup-20260114-120000

# Restore stream configs
for stream_file in nats/stream_*.json; do
  STREAM_NAME=$(jq -r '.config.name' "$stream_file")
  jq '.config' "$stream_file" | nats stream add "$STREAM_NAME" --config /dev/stdin
done
 
# Restore consumer configs
for consumer_file in nats/consumer_*.json; do
  STREAM_NAME=$(jq -r '.stream_name' "$consumer_file")
  CONSUMER_NAME=$(jq -r '.config.durable_name' "$consumer_file")
  jq '.config' "$consumer_file" | nats consumer add "$STREAM_NAME" "$CONSUMER_NAME" --config /dev/stdin
done

kubectl cp nats-backup.tar.gz isa-cloud-staging/nats-0:/tmp/
kubectl exec -n isa-cloud-staging deploy/nats -- \
  tar xzf /tmp/nats-backup.tar.gz -C /
kubectl rollout restart statefulset/nats -n isa-cloud-staging

# From atomic snapshot (preferred)
curl -sf -X PUT --data-binary @consul-snapshot.snap http://localhost:8500/v1/snapshot
 
# Or via consul CLI
consul snapshot restore /tmp/consul-backup.snap
 
# From KV JSON export
consul kv import @consul-kv-export.json

# Copy persistence file back and restart the broker
kubectl cp mqtt-backup/mosquitto.db \
  isa-cloud-staging/<mosquitto-pod>:/mosquitto/data/mosquitto.db
kubectl rollout restart deployment/mosquitto -n isa-cloud-staging

# Enabled automatically when replicas > 1
podDisruptionBudget:
  enabled: true
  minAvailable: 1        # or use maxUnavailable instead
  # maxUnavailable: 1    # alternative: allow at most 1 pod down

# values.yaml for a service
name: auth-service
namespace: isa-cloud-production
replicas: 3
 
podDisruptionBudget:
  enabled: true
  minAvailable: 2      # Always keep at least 2 pods running during disruptions

# In another repo's workflow
jobs:
  security:
    uses: xenoISA/isA_Cloud/.github/workflows/security-scan.yaml@main
    with:
      python_path: '.'
      go_path: '.'

Vault (HA, 3 replicas)
  └── KV v2 engine at secret/
      └── secret/data/isa-cloud/production/<service>
            ├── postgresql  (password, replication-password, pgpool-admin-password)
            ├── redis       (password)
            ├── neo4j       (password)
            ├── minio       (root-user, root-password)
            ├── emqx        (dashboard-password)
            └── apisix      (admin-key)

External Secrets Operator
  └── ClusterSecretStore (vault-backend)
      └── ExternalSecret CRs (one per service)
          └── K8s Secrets (postgresql-secret, redis-secret, etc.)

# 1. Deploy Vault
helm install vault hashicorp/vault -n isa-cloud-production \
  -f production/values/vault.yaml
 
# 2. Initialize and unseal Vault (one-time)
./production/scripts/vault-init.sh
 
# 3. Deploy External Secrets Operator
helm install external-secrets external-secrets/external-secrets \
  -n external-secrets --create-namespace \
  -f production/values/external-secrets.yaml
 
# 4. Apply the ClusterSecretStore and ExternalSecret CRs
kubectl apply -f production/manifests/cluster-secret-store.yaml
kubectl apply -f production/manifests/external-secrets.yaml

# Check Vault status
./production/scripts/vault-init.sh status
 
# Unseal Vault after pod restart
./production/scripts/vault-init.sh unseal
 
# Update or add secrets
./production/scripts/vault-init.sh seed
 
# Verify ExternalSecrets are syncing
kubectl get externalsecrets -n isa-cloud-production
 
# Check a specific synced secret
kubectl get secret postgresql-secret -n isa-cloud-production -o yaml

# Request rate
sum(rate(apisix_http_status[5m])) by (service)

# Error rate
sum(rate(apisix_http_status{code=~"5.."}[5m])) / sum(rate(apisix_http_status[5m]))

# Latency P99
histogram_quantile(0.99, sum(rate(apisix_http_latency_bucket[5m])) by (le, service))

# Service errors
{namespace="isa-cloud-staging", app="auth-service"} |= "error"

# gRPC requests
{namespace="isa-cloud-staging"} | json | method=~".*"

# Slow requests
{namespace="isa-cloud-staging"} | json | duration > 1s

# Check all pods
kubectl get pods -n isa-cloud-staging
 
# Check specific service
kubectl describe pod auth-service-xxx -n isa-cloud-staging
 
# View logs
kubectl logs -f auth-service-xxx -n isa-cloud-staging

# All services
curl http://localhost:8500/v1/catalog/services | jq
 
# Service health
curl http://localhost:8500/v1/health/service/auth_service | jq
 
# Critical services
curl http://localhost:8500/v1/health/state/critical | jq

grpcurl -plaintext localhost:50061 grpc.health.v1.Health/Check

# Check events
kubectl get events -n isa-cloud-staging --sort-by='.lastTimestamp'
 
# Describe pod
kubectl describe pod <pod-name> -n isa-cloud-staging
 
# Check resource limits
kubectl top pods -n isa-cloud-staging

# Check service exists
kubectl get svc -n isa-cloud-staging
 
# Check endpoints
kubectl get endpoints <service-name> -n isa-cloud-staging
 
# Test DNS
kubectl run debug --rm -it --image=busybox -- nslookup <service-name>

kubectl rollout restart deployment/<service-name> -n isa-cloud-staging
kubectl rollout status deployment/<service-name> -n isa-cloud-staging

kubectl exec -it redis-master-0 -n isa-cloud-staging -- redis-cli FLUSHDB