Chapter 3: Argo CD Advanced Configuration

Key Concepts

ApplicationSet Overview

ApplicationSet is an extension of Argo CD for automating the generation and management of multiple Applications. It supports batch creation of applications from templates, suitable for multi-environment, multi-cluster, and multi-tenant scenarios.

flowchart TB
    subgraph Traditional["Traditional Approach"]
        App1[Application 1]
        App2[Application 2]
        App3[Application 3]
        AppN[Application N...]
    end

    subgraph ApplicationSet["ApplicationSet Approach"]
        AS[ApplicationSet<br/>Template + Generator]
        AS --> A1[App Dev]
        AS --> A2[App Staging]
        AS --> A3[App Production]
    end

Generator Types

mindmap
  root((Generators))
    List
      Static list
      Hard-coded params
    Cluster
      Cluster discovery
      Auto-register
    Git
      Directory structure
      File content
    Matrix
      Combined generation
      Multi-dimensional
    Merge
      Merge parameters
      Override config
    SCM Provider
      GitHub/GitLab
      Repository discovery
    Pull Request
      PR preview env
      Auto cleanup

ApplicationSet Generators

List Generator

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: multi-env-apps
  namespace: argocd
spec:
  generators:
  - list:
      elements:
      - env: development
        namespace: dev
        replicas: "1"
        cluster: https://kubernetes.default.svc
      - env: staging
        namespace: staging
        replicas: "2"
        cluster: https://kubernetes.default.svc
      - env: production
        namespace: production
        replicas: "5"
        cluster: https://prod-cluster.example.com

  template:
    metadata:
      name: 'myapp-{{env}}'
      labels:
        environment: '{{env}}'
    spec:
      project: default
      source:
        repoURL: https://github.com/myorg/myapp.git
        targetRevision: HEAD
        path: 'overlays/{{env}}'
        kustomize:
          images:
            - myapp=myregistry.com/myapp:{{env}}-latest
      destination:
        server: '{{cluster}}'
        namespace: '{{namespace}}'
      syncPolicy:
        automated:
          prune: true
          selfHeal: true
        syncOptions:
          - CreateNamespace=true

Cluster Generator

# Automatically create applications for all registered clusters
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: cluster-apps
  namespace: argocd
spec:
  generators:
  - clusters:
      # Select clusters with specific labels
      selector:
        matchLabels:
          env: production
      # Or use match expressions
      # selector:
      #   matchExpressions:
      #   - key: env
      #     operator: In
      #     values:
      #     - production
      #     - staging

  template:
    metadata:
      name: 'monitoring-{{name}}'
    spec:
      project: default
      source:
        repoURL: https://github.com/myorg/monitoring.git
        targetRevision: HEAD
        path: manifests
      destination:
        server: '{{server}}'
        namespace: monitoring
      syncPolicy:
        automated:
          prune: true
          selfHeal: true

# Add labels to clusters
argocd cluster set <cluster-name> --label env=production

Git Generator - Directory

# Generate applications based on Git directory structure
# Repository structure:
# apps/
# ├── app1/
# │   └── kustomization.yaml
# ├── app2/
# │   └── kustomization.yaml
# └── app3/
#     └── kustomization.yaml

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: git-directory-apps
  namespace: argocd
spec:
  generators:
  - git:
      repoURL: https://github.com/myorg/gitops-config.git
      revision: HEAD
      directories:
      - path: apps/*
      # Exclude specific directories
      # - path: apps/deprecated
      #   exclude: true

  template:
    metadata:
      name: '{{path.basename}}'
    spec:
      project: default
      source:
        repoURL: https://github.com/myorg/gitops-config.git
        targetRevision: HEAD
        path: '{{path}}'
      destination:
        server: https://kubernetes.default.svc
        namespace: '{{path.basename}}'
      syncPolicy:
        automated:
          prune: true
          selfHeal: true
        syncOptions:
          - CreateNamespace=true

Git Generator - File

# Generate applications based on configuration files
# Repository structure:
# config/
# ├── dev.json
# ├── staging.json
# └── prod.json
#
# dev.json content:
# {
#   "env": "development",
#   "replicas": 1,
#   "domain": "dev.example.com"
# }

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: git-file-apps
  namespace: argocd
spec:
  generators:
  - git:
      repoURL: https://github.com/myorg/gitops-config.git
      revision: HEAD
      files:
      - path: "config/*.json"

  template:
    metadata:
      name: 'myapp-{{env}}'
      annotations:
        app.kubernetes.io/domain: '{{domain}}'
    spec:
      project: default
      source:
        repoURL: https://github.com/myorg/myapp.git
        targetRevision: HEAD
        path: manifests
        helm:
          parameters:
          - name: replicas
            value: '{{replicas}}'
          - name: ingress.host
            value: '{{domain}}'
      destination:
        server: https://kubernetes.default.svc
        namespace: '{{env}}'
      syncPolicy:
        automated:
          prune: true
          selfHeal: true

Matrix Generator

# Combine multiple generators (Cartesian product)
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: matrix-apps
  namespace: argocd
spec:
  generators:
  - matrix:
      generators:
      # First dimension: clusters
      - clusters:
          selector:
            matchLabels:
              type: workload
      # Second dimension: application list
      - list:
          elements:
          - app: frontend
            port: "80"
          - app: backend
            port: "8080"
          - app: cache
            port: "6379"

  # Generated combinations:
  # cluster1 x frontend, cluster1 x backend, cluster1 x cache
  # cluster2 x frontend, cluster2 x backend, cluster2 x cache
  # ...

  template:
    metadata:
      name: '{{name}}-{{app}}'
    spec:
      project: default
      source:
        repoURL: https://github.com/myorg/apps.git
        targetRevision: HEAD
        path: '{{app}}'
      destination:
        server: '{{server}}'
        namespace: '{{app}}'
      syncPolicy:
        automated:
          prune: true
          selfHeal: true

Merge Generator

# Merge outputs from multiple generators
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: merge-apps
  namespace: argocd
spec:
  generators:
  - merge:
      mergeKeys:
      - env
      generators:
      # Base configuration
      - list:
          elements:
          - env: dev
            replicas: "1"
            resources: small
          - env: staging
            replicas: "2"
            resources: medium
          - env: prod
            replicas: "5"
            resources: large
      # Override specific environment configuration
      - list:
          elements:
          - env: prod
            replicas: "10"  # Override prod replicas

  template:
    metadata:
      name: 'myapp-{{env}}'
    spec:
      project: default
      source:
        repoURL: https://github.com/myorg/myapp.git
        targetRevision: HEAD
        path: overlays/{{env}}
        kustomize:
          replicas:
          - name: myapp
            count: '{{replicas}}'
      destination:
        server: https://kubernetes.default.svc
        namespace: '{{env}}'

Pull Request Generator

# Create preview environment for each PR
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: pr-preview
  namespace: argocd
spec:
  generators:
  - pullRequest:
      github:
        owner: myorg
        repo: myapp
        tokenRef:
          secretName: github-token
          key: token
        labels:
        - preview
      requeueAfterSeconds: 60

  template:
    metadata:
      name: 'pr-{{number}}-{{branch_slug}}'
      labels:
        pr-number: '{{number}}'
    spec:
      project: default
      source:
        repoURL: https://github.com/myorg/myapp.git
        targetRevision: '{{head_sha}}'
        path: manifests
        kustomize:
          namePrefix: pr-{{number}}-
          commonLabels:
            pr: '{{number}}'
      destination:
        server: https://kubernetes.default.svc
        namespace: 'pr-{{number}}'
      syncPolicy:
        automated:
          prune: true
          selfHeal: true
        syncOptions:
          - CreateNamespace=true

Multi-Cluster Management

Cluster Architecture

flowchart TB
    subgraph Management["Management Cluster"]
        ArgoCD[Argo CD]
    end

    subgraph Workload["Workload Clusters"]
        Dev[Development Cluster]
        Staging[Staging Cluster]
        Prod1[Production Cluster 1]
        Prod2[Production Cluster 2]
    end

    ArgoCD -->|Manages| Dev
    ArgoCD -->|Manages| Staging
    ArgoCD -->|Manages| Prod1
    ArgoCD -->|Manages| Prod2

Adding Clusters

# Add cluster using kubeconfig context
argocd cluster add production-context \
  --name production \
  --label env=production \
  --label region=us-east-1

# List clusters
argocd cluster list

# View cluster details
argocd cluster get production

# Remove cluster
argocd cluster rm production

Cluster Secret Configuration

apiVersion: v1
kind: Secret
metadata:
  name: production-cluster
  namespace: argocd
  labels:
    argocd.argoproj.io/secret-type: cluster
    env: production
    region: us-east-1
stringData:
  name: production
  server: https://production.k8s.example.com:6443
  config: |
    {
      "bearerToken": "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9...",
      "tlsClientConfig": {
        "insecure": false,
        "caData": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0t..."
      }
    }

Multi-Cluster Application Deployment

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: myapp-production
  namespace: argocd
spec:
  project: default
  source:
    repoURL: https://github.com/myorg/myapp.git
    targetRevision: v1.2.3
    path: overlays/production
  destination:
    # Use cluster name
    name: production
    # Or use server URL
    # server: https://production.k8s.example.com:6443
    namespace: myapp
  syncPolicy:
    automated:
      prune: true
      selfHeal: true

Sync Policy Details

Sync Options

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: myapp
  namespace: argocd
spec:
  syncPolicy:
    automated:
      prune: true           # Delete resources not in Git
      selfHeal: true        # Automatically fix cluster state drift
      allowEmpty: false     # Don't allow application to become empty

    syncOptions:
      # Namespace
      - CreateNamespace=true    # Automatically create namespace

      # Resource validation
      - Validate=true           # Validate resource configuration
      - FailOnSharedResource=true  # Fail on shared resource conflict

      # Sync behavior
      - ApplyOutOfSyncOnly=true # Only sync changed resources
      - PruneLast=true          # Execute delete operations last
      - Replace=false           # Use patch instead of replace

      # Server-side apply
      - ServerSideApply=true    # Use server-side apply
      - RespectIgnoreDifferences=true  # Ignore differences during apply

      # Special resources
      - PrunePropagationPolicy=foreground  # Delete propagation policy

    retry:
      limit: 5
      backoff:
        duration: 5s
        factor: 2
        maxDuration: 3m

Sync Phases and Hooks

flowchart LR
    subgraph PreSync["PreSync Phase"]
        PS1[Database Migration]
        PS2[Config Check]
    end

    subgraph Sync["Sync Phase"]
        S1[Deploy Resources]
    end

    subgraph PostSync["PostSync Phase"]
        POST1[Notification]
        POST2[Tests]
    end

    subgraph SyncFail["SyncFail Phase"]
        F1[Rollback]
        F2[Alert]
    end

    PreSync --> Sync
    Sync --> PostSync
    Sync -->|Failed| SyncFail

# Sync hook examples
apiVersion: batch/v1
kind: Job
metadata:
  name: db-migration
  annotations:
    argocd.argoproj.io/hook: PreSync
    argocd.argoproj.io/hook-delete-policy: HookSucceeded
spec:
  template:
    spec:
      containers:
      - name: migrate
        image: myapp/migrate:latest
        command: ["./migrate", "up"]
      restartPolicy: Never
  backoffLimit: 2

---
apiVersion: batch/v1
kind: Job
metadata:
  name: integration-test
  annotations:
    argocd.argoproj.io/hook: PostSync
    argocd.argoproj.io/hook-delete-policy: BeforeHookCreation
spec:
  template:
    spec:
      containers:
      - name: test
        image: myapp/test:latest
        command: ["./run-tests.sh"]
      restartPolicy: Never
  backoffLimit: 1

---
apiVersion: batch/v1
kind: Job
metadata:
  name: slack-notify
  annotations:
    argocd.argoproj.io/hook: SyncFail
    argocd.argoproj.io/hook-delete-policy: HookSucceeded
spec:
  template:
    spec:
      containers:
      - name: notify
        image: curlimages/curl
        command:
        - curl
        - -X
        - POST
        - -d
        - '{"text":"Sync failed for myapp"}'
        - https://hooks.slack.com/services/xxx
      restartPolicy: Never

Hook Delete Policies

Ignore Differences Configuration

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: myapp
  namespace: argocd
spec:
  ignoreDifferences:
  # Ignore Deployment replicas (managed by HPA)
  - group: apps
    kind: Deployment
    jsonPointers:
    - /spec/replicas

  # Ignore Service clusterIP
  - group: ""
    kind: Service
    jsonPointers:
    - /spec/clusterIP
    - /spec/clusterIPs

  # Ignore specific field in specific resource
  - group: apps
    kind: Deployment
    name: myapp
    namespace: production
    jsonPointers:
    - /spec/template/spec/containers/0/image

  # Use JQ expression
  - group: ""
    kind: Secret
    jqPathExpressions:
    - '.data["tls.crt"]'
    - '.data["tls.key"]'

  # Ignore annotation in all resources
  - group: "*"
    kind: "*"
    managedFieldsManagers:
    - kube-controller-manager

Health Checks

Built-in Health Checks

flowchart TB
    subgraph Resources["Resource Types"]
        Deploy[Deployment]
        STS[StatefulSet]
        DS[DaemonSet]
        SVC[Service]
        Ingress[Ingress]
        PVC[PVC]
        Job[Job]
    end

    subgraph Health["Health Status"]
        Healthy[Healthy]
        Progressing[Progressing]
        Degraded[Degraded]
        Suspended[Suspended]
    end

    Deploy -->|All Pods Ready| Healthy
    Deploy -->|Deploying| Progressing
    Deploy -->|Pods Failed| Degraded

    STS -->|All Pods Ready| Healthy
    Job -->|Completed| Healthy
    Job -->|Running| Progressing
    Job -->|Failed| Degraded

Custom Health Checks

# argocd-cm ConfigMap
apiVersion: v1
kind: ConfigMap
metadata:
  name: argocd-cm
  namespace: argocd
data:
  # Custom health check script (Lua)
  resource.customizations.health.argoproj.io_Rollout: |
    hs = {}
    if obj.status ~= nil then
      if obj.status.phase == "Healthy" then
        hs.status = "Healthy"
        hs.message = "Rollout is healthy"
      elseif obj.status.phase == "Paused" then
        hs.status = "Suspended"
        hs.message = "Rollout is paused"
      elseif obj.status.phase == "Progressing" then
        hs.status = "Progressing"
        hs.message = "Rollout is progressing"
      else
        hs.status = "Degraded"
        hs.message = obj.status.message
      end
    else
      hs.status = "Progressing"
      hs.message = "Waiting for rollout status"
    end
    return hs

  # Custom CRD health check
  resource.customizations.health.mycrd.example.com_MyResource: |
    hs = {}
    if obj.status ~= nil and obj.status.conditions ~= nil then
      for i, condition in ipairs(obj.status.conditions) do
        if condition.type == "Ready" and condition.status == "True" then
          hs.status = "Healthy"
          hs.message = condition.message
          return hs
        end
      end
    end
    hs.status = "Progressing"
    hs.message = "Waiting for Ready condition"
    return hs

Notification Configuration

Installing Argo CD Notifications

# Already included in standard installation
# Configure notifications
kubectl edit configmap argocd-notifications-cm -n argocd

Configuring Notification Templates

apiVersion: v1
kind: ConfigMap
metadata:
  name: argocd-notifications-cm
  namespace: argocd
data:
  # Service configuration
  service.slack: |
    token: $slack-token

  service.webhook.custom: |
    url: https://webhook.example.com
    headers:
    - name: Authorization
      value: Bearer $webhook-token

  # Template definitions
  template.app-deployed: |
    message: |
      Application {{.app.metadata.name}} has been deployed!
      Sync Status: {{.app.status.sync.status}}
      Health Status: {{.app.status.health.status}}
      Revision: {{.app.status.sync.revision}}
    slack:
      attachments: |
        [{
          "color": "#18be52",
          "title": "{{.app.metadata.name}} deployed",
          "fields": [
            {"title": "Sync Status", "value": "{{.app.status.sync.status}}", "short": true},
            {"title": "Health", "value": "{{.app.status.health.status}}", "short": true}
          ]
        }]

  template.app-sync-failed: |
    message: |
      Application {{.app.metadata.name}} sync failed!
      Error: {{.app.status.operationState.message}}
    slack:
      attachments: |
        [{
          "color": "#ff0000",
          "title": "{{.app.metadata.name}} sync failed",
          "text": "{{.app.status.operationState.message}}"
        }]

  # Trigger definitions
  trigger.on-deployed: |
    - when: app.status.operationState.phase in ['Succeeded'] and app.status.health.status == 'Healthy'
      send: [app-deployed]

  trigger.on-sync-failed: |
    - when: app.status.operationState.phase in ['Failed', 'Error']
      send: [app-sync-failed]

  trigger.on-health-degraded: |
    - when: app.status.health.status == 'Degraded'
      send: [app-sync-failed]

Subscribing to Notifications

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: myapp
  namespace: argocd
  annotations:
    # Subscribe to Slack notifications
    notifications.argoproj.io/subscribe.on-deployed.slack: deployments
    notifications.argoproj.io/subscribe.on-sync-failed.slack: alerts
    # Subscribe to webhook
    notifications.argoproj.io/subscribe.on-deployed.custom: ""
spec:
  # ...

Practical Exercise

Multi-Environment GitOps Configuration

# Repository structure
# gitops-config/
# ├── apps/
# │   ├── base/
# │   │   ├── deployment.yaml
# │   │   ├── service.yaml
# │   │   └── kustomization.yaml
# │   └── overlays/
# │       ├── dev/
# │       │   └── kustomization.yaml
# │       ├── staging/
# │       │   └── kustomization.yaml
# │       └── prod/
# │           └── kustomization.yaml
# └── applicationsets/
#     └── multi-env.yaml

# applicationsets/multi-env.yaml
apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: myapp-environments
  namespace: argocd
spec:
  generators:
  - list:
      elements:
      - env: dev
        cluster: https://kubernetes.default.svc
        autoSync: "true"
      - env: staging
        cluster: https://kubernetes.default.svc
        autoSync: "true"
      - env: prod
        cluster: https://production.k8s.example.com
        autoSync: "false"

  template:
    metadata:
      name: 'myapp-{{env}}'
      finalizers:
        - resources-finalizer.argocd.argoproj.io
    spec:
      project: default
      source:
        repoURL: https://github.com/myorg/gitops-config.git
        targetRevision: HEAD
        path: 'apps/overlays/{{env}}'
      destination:
        server: '{{cluster}}'
        namespace: 'myapp-{{env}}'
      syncPolicy:
        automated:
          prune: '{{autoSync}}'
          selfHeal: '{{autoSync}}'
        syncOptions:
          - CreateNamespace=true

  strategy:
    type: RollingSync
    rollingSync:
      steps:
      - matchExpressions:
        - key: env
          operator: In
          values:
          - dev
      - matchExpressions:
        - key: env
          operator: In
          values:
          - staging
      - matchExpressions:
        - key: env
          operator: In
          values:
          - prod

# Apply ApplicationSet
kubectl apply -f applicationsets/multi-env.yaml

# View generated Applications
argocd app list

# View ApplicationSet status
kubectl get applicationset -n argocd
kubectl describe applicationset myapp-environments -n argocd

Summary

Through this chapter, you should have mastered:

• ApplicationSet: Use cases and configuration for various generators
• Multi-Cluster Management: Adding and managing multiple Kubernetes clusters
• Sync Policies: Sync options, hooks, and retry configuration
• Health Checks: Built-in checks and custom Lua scripts
• Notification System: Configuring Slack, webhooks, and other notifications

In the next chapter, we will learn about Argo Workflows and master Kubernetes-native workflow orchestration.