# Docker Compose — Stack local Big Data + IA
# Componentes:
#   - MinIO (S3 compatible storage)
#   - Polaris (Iceberg REST catalog) — placeholder, ver nota
#   - Spark master + worker (con Iceberg + S3A)
#   - Trino (con Iceberg connector)
#   - Postgres (operacional + pgvector + catálogo dbt)
#   - Kafka (KRaft mode, sin Zookeeper)
#   - Schema Registry
#   - Kafka Connect (Debezium)
#   - Kafka UI
#   - Flink JobManager + TaskManager
#   - Airflow (standalone)
#   - MLflow tracking server
#   - DataHub (separar a compose dedicado, ver docs/04)
#
# Memoria recomendada: 16GB+ Docker; 32GB host.
# Levantar todo: docker compose up -d
# Apagar y limpiar: docker compose down -v

version: "3.9"

x-airflow-common: &airflow-common
  image: apache/airflow:2.10.5-python3.11
  environment: &airflow-env
    AIRFLOW__CORE__EXECUTOR: LocalExecutor
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres:5432/airflow
    AIRFLOW__CORE__LOAD_EXAMPLES: "false"
    AIRFLOW__CORE__DAGS_FOLDER: /opt/airflow/dags
    AIRFLOW__WEBSERVER__EXPOSE_CONFIG: "true"
    _PIP_ADDITIONAL_REQUIREMENTS: "apache-airflow-providers-trino==5.7.0 dbt-core==1.8.0 dbt-trino==1.8.0 pyiceberg[s3fs]==0.7.1 soda-core-trino==3.3.0"
  volumes:
    - ../pipelines/orchestration/airflow/dags:/opt/airflow/dags
    - ../pipelines/transformation/dbt:/opt/airflow/dbt
    - ../governance/quality/soda:/opt/airflow/soda
  depends_on:
    postgres:
      condition: service_healthy

services:
  # ────────────────────────────────────────────────
  # Storage
  # ────────────────────────────────────────────────
  minio:
    image: minio/minio:RELEASE.2025-04-22T22-12-26Z
    command: server /data --console-address ":9001"
    ports:
      - "9000:9000"   # S3 API
      - "9001:9001"   # console
    environment:
      MINIO_ROOT_USER: admin
      MINIO_ROOT_PASSWORD: admin12345
    volumes:
      - minio_data:/data
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 5s
      retries: 10

  minio-init:
    image: minio/mc:RELEASE.2025-04-16T18-13-26Z
    depends_on:
      minio:
        condition: service_healthy
    entrypoint: >
      /bin/sh -c "
      mc alias set local http://minio:9000 admin admin12345 &&
      mc mb -p local/lakehouse || true &&
      mc mb -p local/warehouse || true &&
      mc mb -p local/checkpoints || true &&
      mc mb -p local/mlflow || true &&
      mc anonymous set download local/lakehouse || true
      "

  # ────────────────────────────────────────────────
  # Postgres (operacional + Airflow + pgvector + MLflow backend)
  # ────────────────────────────────────────────────
  postgres:
    image: pgvector/pgvector:pg16
    environment:
      POSTGRES_USER: admin
      POSTGRES_PASSWORD: admin12345
      POSTGRES_DB: platform
    ports:
      - "5432:5432"
    volumes:
      - postgres_data:/var/lib/postgresql/data
      - ./postgres-init:/docker-entrypoint-initdb.d
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U admin"]
      interval: 5s
      retries: 10

  # ────────────────────────────────────────────────
  # Iceberg REST Catalog
  # Nota: imagen oficial Apache Polaris en preview;
  # usamos Tabular REST Catalog (Iceberg JDBC) por estabilidad.
  # ────────────────────────────────────────────────
  iceberg-rest:
    image: tabulario/iceberg-rest:1.6.0
    depends_on:
      minio-init:
        condition: service_completed_successfully
    ports:
      - "8181:8181"
    environment:
      AWS_ACCESS_KEY_ID: admin
      AWS_SECRET_ACCESS_KEY: admin12345
      AWS_REGION: us-east-1
      CATALOG_WAREHOUSE: s3://lakehouse/
      CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO
      CATALOG_S3_ENDPOINT: http://minio:9000
      CATALOG_S3_PATH__STYLE__ACCESS: "true"

  # ────────────────────────────────────────────────
  # Spark (Iceberg + S3A + REST catalog)
  # ────────────────────────────────────────────────
  spark:
    image: tabulario/spark-iceberg:3.5.3_1.6.0
    depends_on:
      iceberg-rest:
        condition: service_started
    ports:
      - "8080:8080"   # Spark UI
      - "10000:10000" # Thrift server
      - "8888:8888"   # Jupyter
    environment:
      AWS_ACCESS_KEY_ID: admin
      AWS_SECRET_ACCESS_KEY: admin12345
      AWS_REGION: us-east-1
      SPARK_OPTS: >
        --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
        --conf spark.sql.catalog.lakehouse=org.apache.iceberg.spark.SparkCatalog
        --conf spark.sql.catalog.lakehouse.type=rest
        --conf spark.sql.catalog.lakehouse.uri=http://iceberg-rest:8181
        --conf spark.sql.catalog.lakehouse.io-impl=org.apache.iceberg.aws.s3.S3FileIO
        --conf spark.sql.catalog.lakehouse.s3.endpoint=http://minio:9000
        --conf spark.sql.catalog.lakehouse.s3.path-style-access=true
        --conf spark.sql.defaultCatalog=lakehouse
    volumes:
      - ../lakehouse:/workspace/lakehouse
      - ../pipelines:/workspace/pipelines

  # ────────────────────────────────────────────────
  # Trino (federated query con Iceberg connector)
  # ────────────────────────────────────────────────
  trino:
    image: trinodb/trino:451
    depends_on:
      iceberg-rest:
        condition: service_started
    ports:
      - "8081:8080"
    volumes:
      - ./trino/catalog:/etc/trino/catalog
      - ./trino/config.properties:/etc/trino/config.properties
    environment:
      AWS_ACCESS_KEY_ID: admin
      AWS_SECRET_ACCESS_KEY: admin12345

  # ────────────────────────────────────────────────
  # Kafka (KRaft mode, sin Zookeeper)
  # ────────────────────────────────────────────────
  kafka:
    image: confluentinc/cp-kafka:7.7.1
    hostname: kafka
    ports:
      - "9092:9092"
      - "9094:9094"
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://kafka:9093,EXTERNAL://0.0.0.0:9094
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,EXTERNAL://localhost:9094
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      CLUSTER_ID: kraft-cluster-001
    volumes:
      - kafka_data:/var/lib/kafka/data

  schema-registry:
    image: confluentinc/cp-schema-registry:7.7.1
    depends_on:
      - kafka
    ports:
      - "8085:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092

  kafka-connect:
    image: debezium/connect:2.7
    depends_on:
      - kafka
      - schema-registry
      - postgres
    ports:
      - "8086:8083"
    environment:
      BOOTSTRAP_SERVERS: kafka:9092
      GROUP_ID: connect-cluster
      CONFIG_STORAGE_TOPIC: connect-configs
      OFFSET_STORAGE_TOPIC: connect-offsets
      STATUS_STORAGE_TOPIC: connect-status
      KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
      VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
      CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081

  kafka-ui:
    image: provectuslabs/kafka-ui:v0.7.2
    depends_on:
      - kafka
      - schema-registry
      - kafka-connect
    ports:
      - "8082:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: local
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:9092
      KAFKA_CLUSTERS_0_SCHEMAREGISTRY: http://schema-registry:8081
      KAFKA_CLUSTERS_0_KAFKACONNECT_0_NAME: debezium
      KAFKA_CLUSTERS_0_KAFKACONNECT_0_ADDRESS: http://kafka-connect:8083

  # ────────────────────────────────────────────────
  # Flink (streaming + Iceberg)
  # ────────────────────────────────────────────────
  flink-jobmanager:
    image: flink:1.20-scala_2.12-java17
    command: jobmanager
    ports:
      - "8083:8081"
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: flink-jobmanager
        state.backend: rocksdb
        state.checkpoints.dir: s3a://checkpoints/flink/
        s3.endpoint: http://minio:9000
        s3.access-key: admin
        s3.secret-key: admin12345
        s3.path.style.access: true

  flink-taskmanager:
    image: flink:1.20-scala_2.12-java17
    command: taskmanager
    depends_on:
      - flink-jobmanager
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: flink-jobmanager
        taskmanager.numberOfTaskSlots: 4
        state.backend: rocksdb
        state.checkpoints.dir: s3a://checkpoints/flink/
        s3.endpoint: http://minio:9000
        s3.access-key: admin
        s3.secret-key: admin12345
        s3.path.style.access: true

  # ────────────────────────────────────────────────
  # Airflow (standalone)
  # ────────────────────────────────────────────────
  airflow-init:
    <<: *airflow-common
    entrypoint: /bin/bash
    command:
      - -c
      - |
        airflow db migrate &&
        airflow users create --username admin --password admin --firstname admin --lastname admin --role Admin --email admin@example.com || true
    restart: "no"

  airflow:
    <<: *airflow-common
    command: standalone
    ports:
      - "8084:8080"
    depends_on:
      airflow-init:
        condition: service_completed_successfully

  # ────────────────────────────────────────────────
  # MLflow
  # ────────────────────────────────────────────────
  mlflow:
    image: ghcr.io/mlflow/mlflow:v2.16.2
    depends_on:
      postgres:
        condition: service_healthy
      minio-init:
        condition: service_completed_successfully
    ports:
      - "5000:5000"
    environment:
      MLFLOW_S3_ENDPOINT_URL: http://minio:9000
      AWS_ACCESS_KEY_ID: admin
      AWS_SECRET_ACCESS_KEY: admin12345
    command:
      - mlflow
      - server
      - --host
      - 0.0.0.0
      - --backend-store-uri
      - postgresql+psycopg2://admin:admin12345@postgres:5432/mlflow
      - --default-artifact-root
      - s3://mlflow/

volumes:
  minio_data:
  postgres_data:
  kafka_data:
