version: 1

defaults:
  maxRetries: 0
  timeoutSeconds: 300

tasks:
  - id: generate-mock-source
    type: PYTHON
    path: jobs/generate_mock_data.py

  - id: spark-clean-customers
    type: PYSPARK
    path: jobs/spark_clean_customers.py
    dependsOn: [generate-mock-source]
    sparkConfig:
      spark.executor.memory: 2g
      spark.executor.cores: "1"

  - id: spark-enrich-customers
    type: PYSPARK
    path: jobs/spark_enrich_customers.py
    dependsOn: [spark-clean-customers]
    sparkConfig:
      spark.executor.memory: 2g
      spark.executor.cores: "1"

  - id: spark-aggregate-customers
    type: PYSPARK
    path: jobs/spark_aggregate_customers.py
    dependsOn: [spark-enrich-customers]

  - id: spark-validate-customers
    type: PYSPARK
    path: jobs/spark_validate_customers.py
    dependsOn: [spark-enrich-customers]

  - id: ingest-customers
    type: PYTHON
    path: jobs/ingest_customers.py
    dependsOn: [spark-validate-customers]

  - id: publish-metrics
    type: PYTHON
    path: jobs/publish_ingestion_metrics.py
    dependsOn: [ingest-customers, spark-aggregate-customers]
    output:
      type: TABLE
      databaseName: analytics
      schemaName: PUBLIC
      tableName: pipeline_metrics
      mode: APPEND
