chore: set up Airflow 3 and dbt BigQuery

.env.example

@@ -11,18 +11,15 @@ KOIN_DATA_AIRFLOW_DB=airflow_metadata
 KOIN_DATA_SUPERSET_DB=superset_metadata
 
 # Airflow
+KOIN_DATA_AIRFLOW_PORT=8080
 AIRFLOW_UID=50000
 AIRFLOW__CORE__EXECUTOR=LocalExecutor
 AIRFLOW__CORE__LOAD_EXAMPLES=false
 AIRFLOW__WEBSERVER__EXPOSE_CONFIG=false
 AIRFLOW__WEBSERVER__SECRET_KEY=change_me
-AIRFLOW_ADMIN_USERNAME=admin
-AIRFLOW_ADMIN_PASSWORD=admin
-AIRFLOW_ADMIN_FIRSTNAME=KOIN
-AIRFLOW_ADMIN_LASTNAME=Admin
-AIRFLOW_ADMIN_EMAIL=admin@example.com
 
 # Superset
+KOIN_DATA_SUPERSET_PORT=8088
 SUPERSET_SECRET_KEY=change_me
 SUPERSET_ADMIN_USERNAME=admin
 SUPERSET_ADMIN_PASSWORD=admin
@@ -31,11 +28,7 @@ SUPERSET_ADMIN_LASTNAME=Admin
 SUPERSET_ADMIN_EMAIL=admin@example.com
 
 # BigQuery/dbt
+DBT_TARGET=dev
+GOOGLE_APPLICATION_CREDENTIALS=/secrets/gcp-service-account.json
 KOIN_DATA_GCP_PROJECT=your-gcp-project
 KOIN_DATA_BIGQUERY_LOCATION=asia-northeast3
-KOIN_DATA_RAW_DATASET=raw_ga4
-KOIN_DATA_STAGING_DATASET=staging
-KOIN_DATA_BRONZE_DATASET=bronze
-KOIN_DATA_SILVER_DATASET=silver
-KOIN_DATA_GOLD_DATASET=gold
-KOIN_DATA_GA4_TABLE_PREFIX=events_

.gitattributes

@@ -0,0 +1 @@
+*.sh text eol=lf

.gitignore

@@ -11,6 +11,7 @@ airflow/**/__pycache__/
 dbt/**/target/
 dbt/**/dbt_packages/
 dbt/**/logs/
+dbt/**/.user.yml
 
 outputs/tmp/
 notebooks/.ipynb_checkpoints/

README.md

@@ -32,8 +32,7 @@ KOIN_DATA/
     plugins/
     logs/
 
-  dbt/
-    koin/          # 초기에는 빈 폴더, 모델은 단계적으로 추가
+  dbt/            # 초기에는 빈 폴더, 모델은 단계적으로 추가
 
   scripts/
   docs/
@@ -67,6 +66,10 @@ docker compose up -d --build
 
 - GitHub repo: `KOIN_DATA`
 - Docker project: `koin-data`
-- dbt project: `koin`
+- dbt workspace: `dbt/`
 - Python, dbt, SQL 식별자: `underscore_case`
 - 환경변수 prefix: `KOIN_DATA_`
+
+## Docker Compose Reference
+
+- [Docker Compose setup](docs/docker-compose.md)

dbt/.gitkeep

@@ -0,0 +1 @@
+

docker-compose.yml

@@ -11,12 +11,10 @@ x-airflow-common: &airflow-common
   environment:
     AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${KOIN_DATA_POSTGRES_USER:-koin}:${KOIN_DATA_POSTGRES_PASSWORD:-change_me}@postgres:5432/${KOIN_DATA_AIRFLOW_DB:-airflow_metadata}
     AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://${KOIN_DATA_POSTGRES_USER:-koin}:${KOIN_DATA_POSTGRES_PASSWORD:-change_me}@postgres:5432/${KOIN_DATA_AIRFLOW_DB:-airflow_metadata}
-    DBT_PROFILES_DIR: /opt/airflow/dbt/koin
   volumes:
     - ./airflow/dags:/opt/airflow/dags
     - ./airflow/plugins:/opt/airflow/plugins
     - ./airflow/logs:/opt/airflow/logs
-    - ./dbt:/opt/airflow/dbt
     - ./secrets:/opt/airflow/secrets:ro
   depends_on:
     postgres:
@@ -44,24 +42,16 @@ services:
 
   airflow-init:
     <<: *airflow-common
-    command: >
-      bash -lc "airflow db migrate &&
-      airflow users create
-      --username \"$${AIRFLOW_ADMIN_USERNAME}\"
-      --password \"$${AIRFLOW_ADMIN_PASSWORD}\"
-      --firstname \"$${AIRFLOW_ADMIN_FIRSTNAME}\"
-      --lastname \"$${AIRFLOW_ADMIN_LASTNAME}\"
-      --role Admin
-      --email \"$${AIRFLOW_ADMIN_EMAIL}\" || true"
+    command: bash -lc "airflow db migrate"
     restart: "no"
 
   airflow-webserver:
     <<: *airflow-common
-    command: webserver
+    command: api-server
     ports:
-      - "8080:8080"
+      - "${KOIN_DATA_AIRFLOW_PORT:-8080}:8080"
     healthcheck:
-      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
+      test: ["CMD", "curl", "--fail", "http://localhost:8080/api/v2/monitor/health"]
       interval: 30s
       timeout: 10s
       retries: 5
@@ -72,6 +62,18 @@ services:
     command: scheduler
     restart: unless-stopped
 
+  dbt:
+    image: ghcr.io/dbt-labs/dbt-bigquery:1.9.latest
+    env_file:
+      - path: .env
+        required: false
+    working_dir: /usr/app
+    volumes:
+      - ./dbt:/usr/app
+      - ./secrets:/secrets:ro
+    command: ["--version"]
+    restart: "no"
+
   superset:
     image: ${KOIN_DATA_SUPERSET_IMAGE:-koin-data-superset:latest}
     build:
@@ -87,7 +89,7 @@ services:
       - ./docker/superset/superset_config.py:/app/pythonpath/superset_config.py:ro
       - ./secrets:/app/secrets:ro
     ports:
-      - "8088:8088"
+      - "${KOIN_DATA_SUPERSET_PORT:-8088}:8088"
     depends_on:
       postgres:
         condition: service_healthy

docker/airflow/Dockerfile

@@ -1,4 +1,6 @@
-FROM apache/airflow:2.9.3-python3.11
+FROM apache/airflow:3.0.1-python3.11
 
 COPY docker/airflow/requirements.txt /requirements.txt
-RUN pip install --no-cache-dir -r /requirements.txt
+RUN pip install --no-cache-dir \
+    --constraint "https://github.com/apache/airflow/constraints-3.0.1/constraints-3.11.txt" \
+    -r /requirements.txt

docker/airflow/requirements.txt

@@ -1,3 +1 @@
-apache-airflow-providers-google==10.22.0
-dbt-bigquery==1.8.2
-astronomer-cosmos==1.6.0
+apache-airflow-providers-google

docker/postgres/init-databases.sh

@@ -1,12 +1,21 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
+# Create service metadata databases during the first Postgres initialization.
 create_database() {
   local database="$1"
-  psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname postgres <<SQL
-SELECT 'CREATE DATABASE ${database}'
-WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '${database}')\gexec
-SQL
+  local exists
+
+  exists="$(
+    psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname postgres --set=database="$database" \
+      --tuples-only --no-align \
+      --command "SELECT 1 FROM pg_database WHERE datname = :'database'"
+  )"
+
+  if [[ "$exists" != "1" ]]; then
+    psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname postgres --set=database="$database" \
+      --command 'CREATE DATABASE :"database"'
+  fi
 }
 
 create_database "${KOIN_DATA_AIRFLOW_DB:-airflow_metadata}"

docs/docker-compose.md

@@ -0,0 +1,148 @@
+# Docker Compose Setup
+
+This document summarizes the local Docker Compose setup for the KOIN data platform.
+
+## Service Roles
+
+| Service | Role | Long-running |
+| --- | --- | --- |
+| `postgres` | Metadata database for Airflow and Superset | Yes |
+| `airflow-webserver` | Airflow 3 API server and UI | Yes |
+| `airflow-scheduler` | DAG scheduling and task dispatching | Yes |
+| `airflow-init` | Airflow metadata migration | No |
+| `dbt` | dbt BigQuery command runner | No |
+| `superset` | BI dashboard service | Yes |
+
+dbt is not exposed as a web service.
+It runs as a command-line container with the local `dbt/` workspace mounted at `/usr/app`.
+
+## Ports
+
+| Service | Host Port | Container Port | Notes |
+| --- | ---: | ---: | --- |
+| Airflow API/UI | `${KOIN_DATA_AIRFLOW_PORT:-8080}` | `8080` | Open `http://localhost:8080` locally by default |
+| Superset UI | `${KOIN_DATA_SUPERSET_PORT:-8088}` | `8088` | Open `http://localhost:8088` locally by default |
+| Postgres | Not published | `5432` | Internal Docker network only |
+| dbt | None | None | dbt runs as a command-line container, not a web server |
+
+If `8080` or `8088` is already used on a developer machine or EC2 host, change only the host-side values in `.env`:
+
+```text
+KOIN_DATA_AIRFLOW_PORT=18080
+KOIN_DATA_SUPERSET_PORT=18088
+```
+
+Postgres does not publish `5432` to the host. This is intentional for local and EC2 security.
+Containers should connect to Postgres with the Docker Compose service address:
+
+```text
+postgres:5432
+```
+
+## Metadata Database Split
+
+The Postgres container is used for service metadata, not for GA4 analytics data.
+
+| Database | Owner | Purpose |
+| --- | --- | --- |
+| `airflow_metadata` | Airflow | DAG runs, task states, schedules, Airflow users |
+| `superset_metadata` | Superset | Dashboard metadata, charts, users, connection metadata |
+
+The databases are created by:
+
+```text
+docker/postgres/init-databases.sh
+```
+
+The database names are controlled by:
+
+```text
+KOIN_DATA_AIRFLOW_DB=airflow_metadata
+KOIN_DATA_SUPERSET_DB=superset_metadata
+```
+
+## Analytics Data Storage
+
+Analytics data is expected to live in BigQuery, not in the metadata Postgres container.
+The initial `dbt/` directory is intentionally empty so project models can be added deliberately later.
+
+## Local Verification
+
+Create a local `.env` file:
+
+```bash
+cp .env.example .env
+```
+
+Build and start the services:
+
+```bash
+docker compose up -d --build
+```
+
+Check service status:
+
+```bash
+docker compose ps
+```
+
+Expected result:
+
+```text
+postgres            healthy/running
+airflow-webserver   running
+airflow-scheduler   running
+superset            running
+```
+
+Useful logs:
+
+```bash
+docker compose logs postgres
+docker compose logs airflow-webserver
+docker compose logs airflow-scheduler
+docker compose logs superset
+```
+
+## Build Notes
+
+The Airflow image is based on `apache/airflow:3.0.1-python3.11`.
+Use the matching Airflow constraints file during `pip install` so provider versions stay compatible with the base Airflow image.
+
+Airflow 3 uses `airflow api-server` instead of the Airflow 2 `airflow webserver` command.
+
+dbt is kept in a separate `ghcr.io/dbt-labs/dbt-bigquery:1.9.latest` container.
+This avoids dependency conflicts between Airflow 3.x provider constraints and dbt BigQuery dependencies.
+
+For Airflow `3.0.1` and Python `3.11`, the Dockerfile uses:
+
+```text
+https://github.com/apache/airflow/constraints-3.0.1/constraints-3.11.txt
+```
+
+Shell scripts mounted into Linux containers must use LF line endings.
+The Postgres initialization script creates the service metadata databases:
+
+```text
+airflow_metadata
+superset_metadata
+```
+
+Keep `*.sh` files as LF to avoid errors such as:
+
+```text
+env: 'bash\r': No such file or directory
+```
+
+## EC2 Security Group
+
+Recommended inbound rules for an EC2 test server:
+
+| Port | Purpose | Recommendation |
+| ---: | --- | --- |
+| `22` | SSH | Restrict to developer IPs |
+| `8080` | Airflow UI | Restrict to team/VPN IPs |
+| `8088` | Superset UI | Restrict to team/VPN IPs |
+| `5432` | Postgres | Do not expose publicly |
+
+Postgres should stay inside the Docker network unless there is a strong operational reason to expose it.