chore: bake DOT into an Airflow worker image

docker/docker-compose-with-airflow.yml

@@ -1,5 +1,5 @@
 #version: '3.1'
-version: '2.1'
+# version: '2.1'
 
 #
 # https://airflow.apache.org/docs/apache-airflow/2.2.1/start/docker.html
@@ -44,7 +44,8 @@ version: '2.1'
 
 x-airflow-common:
   &airflow-common
-  image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.2.4}
+  # image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.2.4}
+  image: apache/airflow:2.2.4-python3.8
   environment:
     &airflow-common-env
     AIRFLOW__CORE__EXECUTOR: CeleryExecutor
@@ -53,7 +54,7 @@ x-airflow-common:
     AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
     AIRFLOW__CORE__FERNET_KEY: ''
     AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
-    AIRFLOW__CORE__LOAD_EXAMPLES: 'true'
+    # AIRFLOW__CORE__LOAD_EXAMPLES: 'true'
     AIRFLOW__CORE__ENABLE_XCOM_PICKLING: 'true'
     PYTHON_BASE_IMAGE: "python:3:8-slim-buster"
     POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
@@ -91,6 +92,15 @@ services:
           - .:/db_dumps
           - ../db/dot:/docker-entrypoint-initdb.d
 
+  appsmith:
+    image: index.docker.io/appsmith/appsmith-ce
+    container_name: appsmith
+    ports:
+      - "82:80"
+      - "446:443"
+    volumes:
+      - ./appsmith/stacks:/appsmith-stacks
+
   # Not needed here, because we deploy DOT to airflow worker
   #dot:
   #  build:
@@ -188,6 +198,10 @@ services:
 
   airflow-worker:
     <<: *airflow-common
+    image: dot-airflow-worker:latest
+    build:
+      context: ..
+      dockerfile: ./docker/dot/Dockerfile
     command: celery worker
     restart: always
 

docker/dot/Dockerfile

@@ -1,23 +1,26 @@
-FROM python:3.8-slim-buster
+# Start with the exact same Airflow image you are already using
+FROM apache/airflow:2.2.4-python3.8
 
-WORKDIR app/
+# Switch to the root user to install system-level dependencies if needed
+USER root
 
-RUN apt-get update
+RUN sed -i 's|deb.debian.org|archive.debian.org|g' /etc/apt/sources.list \
+ && sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list \
+ && sed -i '/buster-updates/d' /etc/apt/sources.list \
+ && printf 'Acquire::Check-Valid-Until "false";\n' > /etc/apt/apt.conf.d/99no-check-valid-until \
+ && rm -f /etc/apt/sources.list.d/pgdg.list \
+ && rm -f /etc/apt/sources.list.d/mssql-release.list \
+ && apt-get update
 
-RUN pip install --upgrade pip
+RUN apt-get -y install postgresql-client
 
-# This uses the requirements as defined in the dot directory
-COPY ./dot/requirements_dot.txt requirements_dot.txt
-COPY ./dot/install_dot.sh install_dot.sh
-RUN ./install_dot.sh
 
-# This will be overwritten if using local mounts, but keeping here for the demo build
-COPY ../ /app
-COPY ./docker/dot/dot_config.yml /app/dot/config/dot_config.yml
-COPY ./dot/config/example/self_tests/dot_config_docker.yml /app/dot/self_tests/data/base_self_test/dot_config.yml
+# Switch back to the 'airflow' user (best practice for Airflow images)
+USER airflow
 
-# Install psql, useful
-RUN apt-get -y install postgresql-client
+# Copy your DOT requirements and install script
+COPY --chown=airflow:root ./dot/requirements_dot.txt /tmp/requirements_dot.txt
+COPY --chown=airflow:root ./dot/install_dot.sh /tmp/install_dot.sh
 
-#CMD [ "python3" ]
-CMD tail -f /dev/null
+# Run your install script
+RUN chmod +x /tmp/install_dot.sh && cd /tmp && ./install_dot.sh

dot/install_dot.sh

@@ -1,7 +1,19 @@
+#!/usr/bin/env bash
 # Simple install script to install Python dependencies, dbt and ge
+set -euo pipefail
 
 # Python packages
-pip install --upgrade pip
+# pip install --upgrade pip
+pip install "pip<22" \
+            "setuptools>=59.1.1,<59.7.0" \
+            "wheel<0.45" \
+            "packaging<22"
+
 # Based on this https://stackoverflow.com/questions/69287269/installing-ruamel-yaml-clib-with-docker.
-pip install -U pip setuptools wheel ruamel.yaml ruamel.yaml.clib==0.2.6
-pip install -r requirements_dot.txt
+# pip install -U pip setuptools wheel ruamel.yaml ruamel.yaml.clib==0.2.6
+pip install "ruamel.yaml" "ruamel.yaml.clib==0.2.6"
+
+pip install -r requirements_dot.txt
+
+# sanity check
+pip check

dot/requirements_dot.txt

@@ -3,16 +3,19 @@ click>=8.0,<8.1
 dbt-core==1.0.0
 dbt-extractor==0.4.0
 dbt-postgres==1.0.0
-decorator==5.1.0
+# decorator==5.1.0
+decorator==4.4.2
 Faker==15.1.1
 faker_airtravel==0.4
 flatten_json==0.1.13
 gdown==4.5.3
-great-expectations==0.12.10
+# great-expectations==0.12.10
+great-expectations==0.14.13
 ipython==8.10.0
 ipython-genutils==0.2.0
 jinja2==2.11.3
 json-rpc==1.13.0
+jsonschema==3.1.1
 jupyter-client==7.1.0
 jupyter-core==4.9.1
 jupyterlab-pygments==0.1.2
@@ -25,23 +28,34 @@ nest-asyncio==1.5.4
 notebook==6.4.12
 numpy==1.22.0
 oauthlib==3.2.2
+openapi-spec-validator==0.2.9
 openpyxl==3.0.9
 pandas==1.3.4
-proto-plus==1.19.8
+packaging<22
+pip<22
+# proto-plus==1.19.8
+proto-plus==1.18.1
 psycopg2-binary==2.9.2
+pydantic==1.9.2
 Pygments==2.10.0
 pylint==2.14.1
 pytest==7.2.0
 pytest-timeout==2.1.0
 python_on_whales==0.53.0
-python-slugify==5.0.2
-PyYAML==6.0
-requests==2.23.0
+# python-slugify==5.0.2
+python-slugify>=3.0.0,<5.0
+# PyYAML==6.0
+PyYAML>=5.1,<6
+# requests==2.23.0
+requests>=2.26.0,<3
 requests-oauthlib==1.3.0
 rsa==4.8
 ruamel.yaml==0.17.17
 ruamel.yaml.clib==0.2.6
-setuptools-scm==6.3.2
+setuptools>=59.1.1,<59.7.0
+# setuptools-scm==6.3.2
+setuptools-scm<6
 SQLAlchemy==1.3.24
 toolz==0.11.2
 typing-extensions==3.10.0.2
+wheel<0.45