dt_cde_airlines/docker-compose.yaml at main · YannickPassa/dt_cde_airlines · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
x-airflow-common: &airflow-common
  build:
    context: .
  env_file: .env
  environment: &airflow-common-env
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql://postgres:password@postgres:5432/airflow
    AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_USERS: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}:Admin
    AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_PASSWORDS_FILE: /opt/airflow/passwords/passwords.json
  volumes:
    - ./custom-dev-entrypoint.sh:/entrypoint.sh
    - airflow_passwords:/opt/airflow/passwords
    - ./dags:/opt/airflow/dags
    - ./dbt:/opt/airflow/dbt
    - ./logs:/opt/airflow/logs
  depends_on: &airflow-common-depends-on
    postgres:
      condition: service_healthy
    redis:
      condition: service_healthy

services:
  postgres:
    container_name: pilotage_postgres
    image: postgres:17
    # Disable some safety switches for a faster postgres: https://www.postgresql.org/docs/current/non-durability.html
    command: -c fsync=off -c full_page_writes=off -c synchronous_commit=off
    environment:
      # Required by the "_/postgres" image
      - POSTGRES_PASSWORD=password
      - POSTGRES_DB=airflow
      # Avoid a log error when starting the container:
      # > Role "root" does not exist.
      # Without this variable, the default Unix account ('root')
      # is used automatically when starting postgres.
      # https://www.postgresql.org/docs/current/libpq-envars.html
      - PGUSER=postgres
      - PGPASSWORD=password
    volumes:
      - postgres_data:/var/lib/postgresql/data
      - ./docker/postgres/initdb.d:/docker-entrypoint-initdb.d
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "postgres"]
      interval: 5s
      retries: 5
    restart: unless-stopped
    ports:
      - "127.0.0.1:${POSTGRES_PORT_ON_DOCKER_HOST:-5432}:5432"

  redis:
    image: redis:7
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 30s
      retries: 50
    restart: unless-stopped

  airflow-init:
    <<: *airflow-common
    container_name: dt_cde_airlines_airflow_init
    command: ["init"]
    restart: "no"
    environment:
      <<: *airflow-common-env
      _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
      _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}

  airflow-webserver:
    <<: *airflow-common
    container_name: dt_cde_airlines_airflow_webserver
    command: ["api-server"]
    restart: unless-stopped
    ports:
      - "127.0.0.1:${AIRFLOW_PORT_ON_DOCKER_HOST:-8080}:8080"
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
      interval: 10s
      timeout: 10s
      retries: 5
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-scheduler:
    <<: *airflow-common
    container_name: dt_cde_airlines_airflow_scheduler
    command: ["scheduler"]
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", 'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"']
      interval: 10s
      timeout: 10s
      retries: 5
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-worker:
    <<: *airflow-common
    container_name: dt_cde_airlines_airflow_worker
    command: ["celery", "worker"]
    restart: unless-stopped
    healthcheck:
      test:
        - "CMD-SHELL"
        - 'celery --app airflow.providers.celery.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
      interval: 10s
      timeout: 10s
      retries: 5
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

volumes:
  postgres_data:
  airflow_passwords: