diff --git a/docker/docker-compose-with-airflow.yml b/docker/docker-compose-with-airflow.yml index 03bce4f..1e9cb50 100644 --- a/docker/docker-compose-with-airflow.yml +++ b/docker/docker-compose-with-airflow.yml @@ -1,5 +1,5 @@ #version: '3.1' -version: '2.1' +# version: '2.1' # # https://airflow.apache.org/docs/apache-airflow/2.2.1/start/docker.html @@ -44,7 +44,8 @@ version: '2.1' x-airflow-common: &airflow-common - image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.2.4} + # image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.2.4} + image: apache/airflow:2.2.4-python3.8 environment: &airflow-common-env AIRFLOW__CORE__EXECUTOR: CeleryExecutor @@ -53,7 +54,7 @@ x-airflow-common: AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0 AIRFLOW__CORE__FERNET_KEY: '' AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' - AIRFLOW__CORE__LOAD_EXAMPLES: 'true' + # AIRFLOW__CORE__LOAD_EXAMPLES: 'true' AIRFLOW__CORE__ENABLE_XCOM_PICKLING: 'true' PYTHON_BASE_IMAGE: "python:3:8-slim-buster" POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} @@ -91,6 +92,15 @@ services: - .:/db_dumps - ../db/dot:/docker-entrypoint-initdb.d + appsmith: + image: index.docker.io/appsmith/appsmith-ce + container_name: appsmith + ports: + - "82:80" + - "446:443" + volumes: + - ./appsmith/stacks:/appsmith-stacks + # Not needed here, because we deploy DOT to airflow worker #dot: # build: @@ -188,6 +198,10 @@ services: airflow-worker: <<: *airflow-common + image: dot-airflow-worker:latest + build: + context: .. + dockerfile: ./docker/dot/Dockerfile command: celery worker restart: always diff --git a/docker/dot/Dockerfile b/docker/dot/Dockerfile index 686e1f4..7ac019d 100644 --- a/docker/dot/Dockerfile +++ b/docker/dot/Dockerfile @@ -1,23 +1,26 @@ -FROM python:3.8-slim-buster +# Start with the exact same Airflow image you are already using +FROM apache/airflow:2.2.4-python3.8 -WORKDIR app/ +# Switch to the root user to install system-level dependencies if needed +USER root -RUN apt-get update +RUN sed -i 's|deb.debian.org|archive.debian.org|g' /etc/apt/sources.list \ + && sed -i 's|security.debian.org|archive.debian.org|g' /etc/apt/sources.list \ + && sed -i '/buster-updates/d' /etc/apt/sources.list \ + && printf 'Acquire::Check-Valid-Until "false";\n' > /etc/apt/apt.conf.d/99no-check-valid-until \ + && rm -f /etc/apt/sources.list.d/pgdg.list \ + && rm -f /etc/apt/sources.list.d/mssql-release.list \ + && apt-get update -RUN pip install --upgrade pip +RUN apt-get -y install postgresql-client -# This uses the requirements as defined in the dot directory -COPY ./dot/requirements_dot.txt requirements_dot.txt -COPY ./dot/install_dot.sh install_dot.sh -RUN ./install_dot.sh -# This will be overwritten if using local mounts, but keeping here for the demo build -COPY ../ /app -COPY ./docker/dot/dot_config.yml /app/dot/config/dot_config.yml -COPY ./dot/config/example/self_tests/dot_config_docker.yml /app/dot/self_tests/data/base_self_test/dot_config.yml +# Switch back to the 'airflow' user (best practice for Airflow images) +USER airflow -# Install psql, useful -RUN apt-get -y install postgresql-client +# Copy your DOT requirements and install script +COPY --chown=airflow:root ./dot/requirements_dot.txt /tmp/requirements_dot.txt +COPY --chown=airflow:root ./dot/install_dot.sh /tmp/install_dot.sh -#CMD [ "python3" ] -CMD tail -f /dev/null +# Run your install script +RUN chmod +x /tmp/install_dot.sh && cd /tmp && ./install_dot.sh diff --git a/dot/install_dot.sh b/dot/install_dot.sh index 5346d9d..160227d 100755 --- a/dot/install_dot.sh +++ b/dot/install_dot.sh @@ -1,7 +1,19 @@ +#!/usr/bin/env bash # Simple install script to install Python dependencies, dbt and ge +set -euo pipefail # Python packages -pip install --upgrade pip +# pip install --upgrade pip +pip install "pip<22" \ + "setuptools>=59.1.1,<59.7.0" \ + "wheel<0.45" \ + "packaging<22" + # Based on this https://stackoverflow.com/questions/69287269/installing-ruamel-yaml-clib-with-docker. -pip install -U pip setuptools wheel ruamel.yaml ruamel.yaml.clib==0.2.6 -pip install -r requirements_dot.txt \ No newline at end of file +# pip install -U pip setuptools wheel ruamel.yaml ruamel.yaml.clib==0.2.6 +pip install "ruamel.yaml" "ruamel.yaml.clib==0.2.6" + +pip install -r requirements_dot.txt + +# sanity check +pip check \ No newline at end of file diff --git a/dot/requirements_dot.txt b/dot/requirements_dot.txt old mode 100644 new mode 100755 index 0f34f27..94f4e68 --- a/dot/requirements_dot.txt +++ b/dot/requirements_dot.txt @@ -3,16 +3,19 @@ click>=8.0,<8.1 dbt-core==1.0.0 dbt-extractor==0.4.0 dbt-postgres==1.0.0 -decorator==5.1.0 +# decorator==5.1.0 +decorator==4.4.2 Faker==15.1.1 faker_airtravel==0.4 flatten_json==0.1.13 gdown==4.5.3 -great-expectations==0.12.10 +# great-expectations==0.12.10 +great-expectations==0.14.13 ipython==8.10.0 ipython-genutils==0.2.0 jinja2==2.11.3 json-rpc==1.13.0 +jsonschema==3.1.1 jupyter-client==7.1.0 jupyter-core==4.9.1 jupyterlab-pygments==0.1.2 @@ -25,23 +28,34 @@ nest-asyncio==1.5.4 notebook==6.4.12 numpy==1.22.0 oauthlib==3.2.2 +openapi-spec-validator==0.2.9 openpyxl==3.0.9 pandas==1.3.4 -proto-plus==1.19.8 +packaging<22 +pip<22 +# proto-plus==1.19.8 +proto-plus==1.18.1 psycopg2-binary==2.9.2 +pydantic==1.9.2 Pygments==2.10.0 pylint==2.14.1 pytest==7.2.0 pytest-timeout==2.1.0 python_on_whales==0.53.0 -python-slugify==5.0.2 -PyYAML==6.0 -requests==2.23.0 +# python-slugify==5.0.2 +python-slugify>=3.0.0,<5.0 +# PyYAML==6.0 +PyYAML>=5.1,<6 +# requests==2.23.0 +requests>=2.26.0,<3 requests-oauthlib==1.3.0 rsa==4.8 ruamel.yaml==0.17.17 ruamel.yaml.clib==0.2.6 -setuptools-scm==6.3.2 +setuptools>=59.1.1,<59.7.0 +# setuptools-scm==6.3.2 +setuptools-scm<6 SQLAlchemy==1.3.24 toolz==0.11.2 typing-extensions==3.10.0.2 +wheel<0.45