I have had a very strange problem when I build my docker service. My service framework is docker+gunicorn+flask+redis+vue,I can build the image and up service normally, but when I login it, the error as follows come out.
**dw-backend-v2 | 2023-01-12 11:04:30,546 exception_handler.py [line: 28] ERROR: Traceback (most recent call last):
dw-backend-v2 | File "/opt/bitnami/python/lib/python3.9/site-packages/redis/connection.py", line 611, in connect
dw-backend-v2 | sock = self.retry.call_with_retry(
dw-backend-v2 | File "/opt/bitnami/python/lib/python3.9/site-packages/redis/retry.py", line 46, in call_with_retry
dw-backend-v2 | return do()
dw-backend-v2 | File "/opt/bitnami/python/lib/python3.9/site-packages/redis/connection.py", line 612, in <lambda>
dw-backend-v2 | lambda: self._connect(), lambda error: self.disconnect(error)
dw-backend-v2 | File "/opt/bitnami/python/lib/python3.9/site-packages/redis/connection.py", line 677, in _connect
dw-backend-v2 | raise err
dw-backend-v2 | File "/opt/bitnami/python/lib/python3.9/site-packages/redis/connection.py", line 665, in _connect
dw-backend-v2 | sock.connect(socket_address)
dw-backend-v2 | OSError: [Errno 99] Cannot assign requested address**
docker-compose:
version: '2'
services:
backend:
build:
context: ./dw_backend
dockerfile: Dockerfile
image: dw-backend:2.0.0
container_name: dw-backend-v2
restart: always
ports:
- "9797:9797"
volumes:
- "./dw_backend:/home/data_warehouse/app"
# privileged: true
environment:
TZ: "Asia/Shanghai"
FLASK_ENV: DEBUG
RD_HOST: redis
RD_PORT: 6379
RD_DB: 2
RD_PWD:
RD_POOL_SIZE: 10
RD_KEY_EXPIRE: 43200
frontend:
hostname: dw-frontend-v2
container_name: dw-frontend-v2
restart: always
build:
context: ./dw_frontend
dockerfile: Dockerfile
ports:
- "8150:80"
- "2443:443"
volumes:
- ./dw_frontend/dist:/usr/share/nginx/html
links:
- backend
depends_on:
- backend
environment:
TZ: "Asia/Shanghai"
networks:
default:
external:
name: common_net
I even deleted all the codes related to redis, but the same error still happened! I could't find any solutions to solve this problem.
And my another one service is normal,all of the services connected same redis service, the difference between service I and service II is service II is restful API but service I is not.
Does anyone know the reason for this problem?
There are also no any ports keep in "TIME_WAIT" status!
I have tried to delete all the code related to redis and even write a new method to connect redis, but no miracle has happened!
I hope someone can help me solve it.
OK，I solved this problem，I add a parameter "extra_hosts: localhost: redis" in my docker-compose.yml file, it's solved !
I have a script writen in a python:
#!/usr/bin/env python
import time
import rediswq
from datetime import datetime
import os
#import subprocess
#host="redis"
# Uncomment next two lines if you do not have Kube-DNS working.
import os
host = os.getenv("REDIS_SERVICE_HOST")
q = rediswq.RedisWQ(name="dealer_codes", host=host)
print("Worker with sessionID: " + q.sessionID())
print("Initial queue state: empty=" + str(q.empty()))
while not q.empty():
item = q.lease(lease_secs=10, block=True, timeout=2)
if item is not None:
itemstr = item.decode("utf-8")
time.sleep(10) # Put your actual work here instead of sleep.
q.complete(item)
else:
print("Waiting for work")
print("Queue empty, exiting")
which is working when I run a redis pod with configuration:
apiVersion: v1
kind: Pod
metadata:
name: redis-master
namespace: sandbox
labels:
app: redis
spec:
containers:
- name: master
image: redis
command:
- redis-server
env:
- name: MASTER
value: "true"
ports:
- containerPort: 6379
when I convert this pod definition file to job like it follows:
apiVersion: batch/v1
kind: Job
metadata:
name: redis-master
namespace: sandbox
spec:
parallelism: 1
ttlSecondsAfterFinished: 10
template:
spec:
containers:
- name: master
image: redis
command:
- redis-server
env:
- name: MASTER
value: "true"
ports:
- containerPort: 6379
restartPolicy: Never
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: sandbox
spec:
ports:
- port: 6379
targetPort: 6379
selector:
app: redis
I get an error from script that says ConnectionRefusedError: [Errno 111] Connection refused:
Worker with sessionID: c7537dc0-0a7c-4d8e-a845-b863441f6433
Traceback (most recent call last):
File "/usr/local/lib/python3.10/site-packages/redis/connection.py", line 614, in connect
sock = self.retry.call_with_retry(
File "/usr/local/lib/python3.10/site-packages/redis/retry.py", line 45, in call_with_retry
return do()
File "/usr/local/lib/python3.10/site-packages/redis/connection.py", line 615, in <lambda>
lambda: self._connect(), lambda error: self.disconnect(error)
File "/usr/local/lib/python3.10/site-packages/redis/connection.py", line 680, in _connect
raise err
File "/usr/local/lib/python3.10/site-packages/redis/connection.py", line 668, in _connect
sock.connect(socket_address)
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "//worker.py", line 16, in <module>
print("Initial queue state: empty=" + str(q.empty()))
File "/rediswq.py", line 56, in empty
return self._main_qsize() == 0 and self._processing_qsize() == 0
File "/rediswq.py", line 45, in _main_qsize
return self._db.llen(self._main_q_key)
File "/usr/local/lib/python3.10/site-packages/redis/commands/core.py", line 2498, in llen
return self.execute_command("LLEN", name)
File "/usr/local/lib/python3.10/site-packages/redis/client.py", line 1215, in execute_command
conn = self.connection or pool.get_connection(command_name, **options)
File "/usr/local/lib/python3.10/site-packages/redis/connection.py", line 1386, in get_connection
connection.connect()
File "/usr/local/lib/python3.10/site-packages/redis/connection.py", line 620, in connect
raise ConnectionError(self._error_message(e))
redis.exceptions.ConnectionError: Error 111 connecting to 10.104.106.102:6379. Connection refused.
I see that is something regarding connection. how can i make this script is working?
have encountered this error "Unrecognized field: schemaType (HTTP status code 422, SR code 422)" when i execute a json_producer.py example in Confluent Github repository
this is my docker-compose:
version: '3.3'
services:
postgres:
container_name: postgres
ports:
- '5432:5432'
environment:
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=postgres
- POSTGRES_DB=shipment_db
- PGPASSWORD=password
image: 'debezium/postgres:13'
zookeeper:
container_name: zookeeper
ports:
- '2181:2181'
- '2888:2888'
- '3888:3888'
image: 'debezium/zookeeper:1.7'
kafka:
container_name: kafka
ports:
- '9092:9092'
links:
- 'zookeeper:zookeeper'
image: 'debezium/kafka:1.7'
environment:
ZOOKEEPER_CONNECT: zookeeper:2181
#volumes:
# - ./kafka:/kafka/config:rw
connect:
image: debezium/connect:1.7
hostname: connect
container_name: connect
ports:
- 8083:8083
environment:
BOOTSTRAP_SERVERS: kafka:9092
GROUP_ID: 1
CONFIG_STORAGE_TOPIC: my_connect_configs
OFFSET_STORAGE_TOPIC: my_connect_offsets
STATUS_STORAGE_TOPIC: my_connect_statuses
CONNECT_BOOTSTRAP_SERVERS: kafka:9092
CONNECT_GROUP_ID: connect-cluster-A
CONNECT_PLUGIN_PATH: /kafka/data, /kafka/connect
#EXTERNAL_LIBS_DIR: /kafka/external_libs,/kafka/data
CLASSPATH: /kafka/data/*
KAFKA_CONNECT_PLUGINS_DIR: /kafka/data, /kafka/connect
#CONNECT_LOG4J_LOGGERS: "org.apache.kafka.connect=DEBUG,org.apache.plc4x.kafka.Plc4xSinkConnector=DEBUG"
volumes:
- type: bind
source: ./plugins
target: /kafka/data
depends_on:
- zookeeper
- kafka
- postgres
links:
- zookeeper
- kafka
- postgres
schema-registry:
image: confluentinc/cp-schema-registry:5.4.6
hostname: schema-registry
container_name: schema-registry
depends_on:
- zookeeper
- kafka
ports:
- "8081:8081"
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
SCHEMA_REGISTRY_LISTENERS: "http://0.0.0.0:8081"
ksqldb-server:
image: confluentinc/ksqldb-server:0.23.1
hostname: ksqldb-server
container_name: ksqldb-server
depends_on:
- kafka
- zookeeper
- schema-registry
ports:
- "8088:8088"
volumes:
- "./confluent-hub-components/:/usr/share/kafka/plugins/"
environment:
KSQL_LISTENERS: "http://0.0.0.0:8088"
KSQL_BOOTSTRAP_SERVERS: "kafka:9092"
KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: "true"
KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: "true"
KSQL_KSQL_CONNECT_URL: http://connect:8083
the code of json_producer is here in the repository.
The error appear when i execute this command:
$ python3 json_producer.py -b 0.0.0.0:9092 -s http://0.0.0.0:8081 -t test
and the stacktrace is the following:
raceback (most recent call last):
File "/home/alessio/fm_v2/python/confluent-kafka-python-master/examples/venv_examples/lib/python3.8/site-packages/confluent_kafka/serializing_producer.py", line 172, in produce
value = self._value_serializer(value, ctx)
File "/home/alessio/fm_v2/python/confluent-kafka-python-master/examples/venv_examples/lib/python3.8/site-packages/confluent_kafka/schema_registry/json_schema.py", line 190, in __call__
self._schema_id = self._registry.register_schema(subject,
File "/home/alessio/fm_v2/python/confluent-kafka-python-master/examples/venv_examples/lib/python3.8/site-packages/confluent_kafka/schema_registry/schema_registry_client.py", line 336, in register_schema
response = self._rest_client.post(
File "/home/alessio/fm_v2/python/confluent-kafka-python-master/examples/venv_examples/lib/python3.8/site-packages/confluent_kafka/schema_registry/schema_registry_client.py", line 127, in post
return self.send_request(url, method='POST', body=body)
File "/home/alessio/fm_v2/python/confluent-kafka-python-master/examples/venv_examples/lib/python3.8/site-packages/confluent_kafka/schema_registry/schema_registry_client.py", line 174, in send_request
raise SchemaRegistryError(response.status_code,
confluent_kafka.schema_registry.error.SchemaRegistryError: Unrecognized field: schemaType (HTTP status code 422, SR code 422)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "json_producer.py", line 172, in <module>
main(parser.parse_args())
File "json_producer.py", line 151, in main
producer.produce(topic=topic, key=str(uuid4()), value=user,
File "/home/alessio/fm_v2/python/confluent-kafka-python-master/examples/venv_examples/lib/python3.8/site-packages/confluent_kafka/serializing_producer.py", line 174, in produce
raise ValueSerializationError(se)
confluent_kafka.error.ValueSerializationError: KafkaError{code=_VALUE_SERIALIZATION,val=-161,str="Unrecognized field: schemaType (HTTP status code 422, SR code 422)"}
where is the problem? Thanks for any reply.
Jsonschema support was not added to the Confluent Schema Registry until version 6.0, and this is why the error reports issues about a schemaType field, because any lower versions of the Registry response/request payload do not know about that field.
Upgrading to at least that version, or using the latest version of the image will solve that error
If you just want to produce JSON, then you don't need the Registry. More details at https://www.confluent.io/blog/kafka-connect-deep-dive-converters-serialization-explained/ .
You can use the regular producer.py example and provide JSON objects as strings on the CLI
I'm making a Discord bot in python, and I get an error when my bot tries to connect with lavalink. I use Docker-compose for the bot and lavalink, and my lavalink client is wavelink. I have checked that lavalink is on the right port.
Does anyone have any idea what's wrong?
My docker-compose.yml:
version: "3.9"
services:
bot:
build:
context: .
dockerfile: Dockerfile
restart: on-failure
depends_on:
lavalink:
condition: service_started
lavalink:
container_name: lavalink
image: fredboat/lavalink:master
volumes:
- ./Lavalink/application.yml:/opt/Lavalink/application.yml
ports:
- 2333
restart: always
My music.py:
class Music(commands.Cog, wavelink.WavelinkMixin, name="music"):
def __init__(self, client):
self.client = client
self.fancy_name = "Music"
if not hasattr(client, 'wavelink'):
self.client.wavelink = wavelink.Client(bot=self.client)
self.client.loop.create_task(self.start_nodes())
async def start_nodes(self):
await self.client.wait_until_ready()
await self.client.wavelink.initiate_node(host='127.0.0.1',
port=2333,
rest_uri='http://127.0.0.1:2333',
password="pass",
identifier='MAIN',
region='us_central')
And my error:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/wavelink/websocket.py", line 76, in _connect
self._websocket = await self._node.session.ws_connect(uri, headers=self.headers, heartbeat=self._node.heartbeat)
File "/usr/local/lib/python3.9/site-packages/aiohttp/client.py", line 721, in _ws_connect
resp = await self.request(method, url,
File "/usr/local/lib/python3.9/site-packages/aiohttp/client.py", line 480, in _request
conn = await self._connector.connect(
File "/usr/local/lib/python3.9/site-packages/aiohttp/connector.py", line 523, in connect
proto = await self._create_connection(req, traces, timeout)
File "/usr/local/lib/python3.9/site-packages/aiohttp/connector.py", line 858, in _create_connection
_, proto = await self._create_direct_connection(
File "/usr/local/lib/python3.9/site-packages/aiohttp/connector.py", line 1004, in _create_direct_connection
raise last_exc
File "/usr/local/lib/python3.9/site-packages/aiohttp/connector.py", line 980, in _create_direct_connection
transp, proto = await self._wrap_create_connection(
File "/usr/local/lib/python3.9/site-packages/aiohttp/connector.py", line 943, in _wrap_create_connection
raise client_error(req.connection_key, exc) from exc
aiohttp.client_exceptions.ClientConnectorError: Cannot connect to host 127.0.0.1:2333 ssl:default [Connect call failed ('127.0.0.1', 2333)]
Your bot container needs to connect to http://lavalink:2333, not itself on 127.0.0.1
You mapping is wrong you declared the port but didn't mapped it to you localhost port.
change this
ports:
- 2333
to
ports:
- "2333:2333"
this will map the port from your docker container running lavalink to your localhost port 2333 or you can assign anyport you want.
ports:
"first_arg:second_arg"
first : localhost port
second: docker container port
I'm trying to run a DAG that calls a docker container and executes a command inside it, but Airflow cannot execute the task. Follows the error launched:
*** Reading local file: /opt/airflow/logs/docker_operator_dag/docker_command_hello/2021-05-26T02:40:13.171571+00:00/2.log
[2021-05-26 02:45:26,001] {taskinstance.py:877} INFO - Dependencies all met for <TaskInstance: docker_operator_dag.docker_command_hello 2021-05-26T02:40:13.171571+00:00 [queued]>
[2021-05-26 02:45:26,030] {taskinstance.py:877} INFO - Dependencies all met for <TaskInstance: docker_operator_dag.docker_command_hello 2021-05-26T02:40:13.171571+00:00 [queued]>
[2021-05-26 02:45:26,031] {taskinstance.py:1068} INFO -
--------------------------------------------------------------------------------
[2021-05-26 02:45:26,031] {taskinstance.py:1069} INFO - Starting attempt 2 of 2
[2021-05-26 02:45:26,032] {taskinstance.py:1070} INFO -
--------------------------------------------------------------------------------
[2021-05-26 02:45:26,060] {taskinstance.py:1089} INFO - Executing <Task(DockerOperator): docker_command_hello> on 2021-05-26T02:40:13.171571+00:00
[2021-05-26 02:45:26,080] {standard_task_runner.py:52} INFO - Started process 67 to run task
[2021-05-26 02:45:26,083] {standard_task_runner.py:76} INFO - Running: ['airflow', 'tasks', 'run', 'docker_operator_dag', 'docker_command_hello', '2021-05-26T02:40:13.171571+00:00', '--job-id', '11', '--pool', 'default_pool', '--raw', '--subdir', 'DAGS_FOLDER/docker_job/docker-job.py', '--cfg-path', '/tmp/tmp3vl5dv7x', '--error-file', '/tmp/tmptazwx_tc']
[2021-05-26 02:45:26,084] {standard_task_runner.py:77} INFO - Job 11: Subtask docker_command_hello
[2021-05-26 02:45:26,181] {logging_mixin.py:104} INFO - Running <TaskInstance: docker_operator_dag.docker_command_hello 2021-05-26T02:40:13.171571+00:00 [running]> on host f1e5cfe4a07f
[2021-05-26 02:45:26,219] {taskinstance.py:1283} INFO - Exporting the following env vars:
AIRFLOW_CTX_DAG_OWNER=airflow
AIRFLOW_CTX_DAG_ID=docker_operator_dag
AIRFLOW_CTX_TASK_ID=docker_command_hello
AIRFLOW_CTX_EXECUTION_DATE=2021-05-26T02:40:13.171571+00:00
AIRFLOW_CTX_DAG_RUN_ID=manual__2021-05-26T02:40:13.171571+00:00
[2021-05-26 02:45:26,227] {taskinstance.py:1482} ERROR - Task failed with exception
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 677, in urlopen
chunked=chunked,
File "/home/airflow/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 392, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/lib/python3.6/http/client.py", line 1287, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/lib/python3.6/http/client.py", line 1333, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.6/http/client.py", line 1282, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.6/http/client.py", line 1042, in _send_output
self.send(msg)
File "/usr/local/lib/python3.6/http/client.py", line 980, in send
self.connect()
File "/home/airflow/.local/lib/python3.6/site-packages/docker/transport/unixconn.py", line 43, in connect
sock.connect(self.unix_socket)
ConnectionRefusedError: [Errno 111] Connection refused
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.6/site-packages/requests/adapters.py", line 449, in send
timeout=timeout
File "/home/airflow/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 727, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/home/airflow/.local/lib/python3.6/site-packages/urllib3/util/retry.py", line 410, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/home/airflow/.local/lib/python3.6/site-packages/urllib3/packages/six.py", line 734, in reraise
raise value.with_traceback(tb)
File "/home/airflow/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 677, in urlopen
chunked=chunked,
File "/home/airflow/.local/lib/python3.6/site-packages/urllib3/connectionpool.py", line 392, in _make_request
conn.request(method, url, **httplib_request_kw)
File "/usr/local/lib/python3.6/http/client.py", line 1287, in request
self._send_request(method, url, body, headers, encode_chunked)
File "/usr/local/lib/python3.6/http/client.py", line 1333, in _send_request
self.endheaders(body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.6/http/client.py", line 1282, in endheaders
self._send_output(message_body, encode_chunked=encode_chunked)
File "/usr/local/lib/python3.6/http/client.py", line 1042, in _send_output
self.send(msg)
File "/usr/local/lib/python3.6/http/client.py", line 980, in send
self.connect()
File "/home/airflow/.local/lib/python3.6/site-packages/docker/transport/unixconn.py", line 43, in connect
sock.connect(self.unix_socket)
urllib3.exceptions.ProtocolError: ('Connection aborted.', ConnectionRefusedError(111, 'Connection refused'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.6/site-packages/docker/api/client.py", line 207, in _retrieve_server_version
return self.version(api_version=False)["ApiVersion"]
File "/home/airflow/.local/lib/python3.6/site-packages/docker/api/daemon.py", line 181, in version
return self._result(self._get(url), json=True)
File "/home/airflow/.local/lib/python3.6/site-packages/docker/utils/decorators.py", line 46, in inner
return f(self, *args, **kwargs)
File "/home/airflow/.local/lib/python3.6/site-packages/docker/api/client.py", line 230, in _get
return self.get(url, **self._set_request_timeout(kwargs))
File "/home/airflow/.local/lib/python3.6/site-packages/requests/sessions.py", line 555, in get
return self.request('GET', url, **kwargs)
File "/home/airflow/.local/lib/python3.6/site-packages/requests/sessions.py", line 542, in request
resp = self.send(prep, **send_kwargs)
File "/home/airflow/.local/lib/python3.6/site-packages/requests/sessions.py", line 655, in send
r = adapter.send(request, **kwargs)
File "/home/airflow/.local/lib/python3.6/site-packages/requests/adapters.py", line 498, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', ConnectionRefusedError(111, 'Connection refused'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.6/site-packages/airflow/models/taskinstance.py", line 1138, in _run_raw_task
self._prepare_and_execute_task_with_callbacks(context, task)
File "/home/airflow/.local/lib/python3.6/site-packages/airflow/models/taskinstance.py", line 1311, in _prepare_and_execute_task_with_callbacks
result = self._execute_task(context, task_copy)
File "/home/airflow/.local/lib/python3.6/site-packages/airflow/models/taskinstance.py", line 1341, in _execute_task
result = task_copy.execute(context=context)
File "/home/airflow/.local/lib/python3.6/site-packages/airflow/providers/docker/operators/docker.py", line 287, in execute
self.cli = self._get_cli()
File "/home/airflow/.local/lib/python3.6/site-packages/airflow/providers/docker/operators/docker.py", line 319, in _get_cli
return APIClient(base_url=self.docker_url, version=self.api_version, tls=tls_config)
File "/home/airflow/.local/lib/python3.6/site-packages/docker/api/client.py", line 190, in __init__
self._version = self._retrieve_server_version()
File "/home/airflow/.local/lib/python3.6/site-packages/docker/api/client.py", line 215, in _retrieve_server_version
'Error while fetching server API version: {0}'.format(e)
docker.errors.DockerException: Error while fetching server API version: ('Connection aborted.', ConnectionRefusedError(111, 'Connection refused'))
[2021-05-26 02:45:26,230] {taskinstance.py:1532} INFO - Marking task as FAILED. dag_id=docker_operator_dag, task_id=docker_command_hello, execution_date=20210526T024013, start_date=20210526T024526, end_date=20210526T024526
[2021-05-26 02:45:26,261] {local_task_job.py:146} INFO - Task exited with return code 1
I'm using the Airflow docker-compose found here https://airflow.apache.org/docs/apache-airflow/stable/start/docker.html and trying to run the following DAG:
from datetime import datetime, timedelta
from airflow import DAG
from airflow.operators.bash_operator import BashOperator
from airflow.operators.docker_operator import DockerOperator
from airflow.operators.python_operator import BranchPythonOperator
from airflow.operators.dummy_operator import DummyOperator
default_args = {
'owner' : 'airflow',
'description' : 'Use of the DockerOperator',
'depend_on_past' : False,
'start_date' : datetime(2021, 5, 1),
'email_on_failure' : False,
'email_on_retry' : False,
'retries' : 1,
'retry_delay' : timedelta(minutes=5)
}
with DAG('docker_operator_dag', default_args=default_args, schedule_interval="5 * * * *", catchup=False) as dag:
start_dag = DummyOperator(
task_id='start_dag'
)
end_dag = DummyOperator(
task_id='end_dag'
)
t1 = BashOperator(
task_id='print_current_date',
bash_command='date'
)
t2 = DockerOperator(
task_id='docker_command_sleep',
image='docker_image_task',
container_name='task___command_sleep',
api_version='auto',
auto_remove=True,
command="/bin/sleep 30",
docker_url="unix://var/run/docker.sock",
network_mode="bridge"
)
t3 = DockerOperator(
task_id='docker_command_hello',
image='docker_image_task',
container_name='task___command_hello',
api_version='auto',
auto_remove=True,
command="/bin/sleep 40",
docker_url="unix://var/run/docker.sock",
network_mode="bridge"
)
t4 = BashOperator(
task_id='print_hello',
bash_command='echo "hello world"'
)
start_dag >> t1
t1 >> t2 >> t4
t1 >> t3 >> t4
t4 >> end_dag
Also, I'm using Windows 10, I've tried adding the following in volumes: - "/var/run/docker.sock:/var/run/docker.sock" and I've succeeded in Ubuntu, but in Windows doesn't.
As requested, follow the docker-compose file:
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
# Basic Airflow cluster configuration for CeleryExecutor with Redis and PostgreSQL.
#
# WARNING: This configuration is for local development. Do not use it in a production deployment.
#
# This configuration supports basic configuration using environment variables or an .env file
# The following variables are supported:
#
# AIRFLOW_IMAGE_NAME - Docker image name used to run Airflow.
# Default: apache/airflow:master-python3.8
# AIRFLOW_UID - User ID in Airflow containers
# Default: 50000
# AIRFLOW_GID - Group ID in Airflow containers
# Default: 50000
# _AIRFLOW_WWW_USER_USERNAME - Username for the administrator account.
# Default: airflow
# _AIRFLOW_WWW_USER_PASSWORD - Password for the administrator account.
# Default: airflow
#
# Feel free to modify this file to suit your needs.
---
version: '3'
x-airflow-common:
&airflow-common
image: ${AIRFLOW_IMAGE_NAME:-apache/airflow:2.0.2}
environment:
&airflow-common-env
AIRFLOW__CORE__EXECUTOR: CeleryExecutor
AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow#postgres/airflow
AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://airflow:airflow#postgres/airflow
AIRFLOW__CELERY__BROKER_URL: redis://:#redis:6379/0
AIRFLOW__CORE__FERNET_KEY: ''
AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL: 5 # Just to have a fast load in the front-end. Do not use it in production with those configurations.
AIRFLOW__API__AUTH_BACKEND: 'airflow.api.auth.backend.basic_auth'
AIRFLOW__CORE__ENABLE_XCOM_PICKLING: 'true' # "_run_image of the DockerOperator returns now a python string, not a byte string" Ref: https://github.com/apache/airflow/issues/13487
volumes:
- ./dags:/opt/airflow/dags
- ./logs:/opt/airflow/logs
- ./plugins:/opt/airflow/plugins
- "/var/run/docker.sock:/var/run/docker.sock" # We will pass the Docker Deamon as a volume to allow the webserver containers start docker images. Ref: https://stackoverflow.com/q/51342810/7024760
user: "${AIRFLOW_UID:-50000}:${AIRFLOW_GID:-50000}"
depends_on:
redis:
condition: service_healthy
postgres:
condition: service_healthy
services:
postgres:
image: postgres:13
environment:
POSTGRES_USER: airflow
POSTGRES_PASSWORD: airflow
POSTGRES_DB: airflow
volumes:
- postgres-db-volume:/var/lib/postgresql/data
healthcheck:
test: ["CMD", "pg_isready", "-U", "airflow"]
interval: 5s
retries: 5
restart: always
redis:
image: redis:latest
ports:
- 6379:6379
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 30s
retries: 50
restart: always
airflow-webserver:
<<: *airflow-common
command: webserver
ports:
- 8080:8080
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
interval: 10s
timeout: 10s
retries: 5
restart: always
airflow-scheduler:
<<: *airflow-common
command: scheduler
restart: always
airflow-worker:
<<: *airflow-common
command: celery worker
restart: always
airflow-init:
<<: *airflow-common
command: version
environment:
<<: *airflow-common-env
_AIRFLOW_DB_UPGRADE: 'true'
_AIRFLOW_WWW_USER_CREATE: 'true'
_AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
_AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
flower:
<<: *airflow-common
command: celery flower
ports:
- 5555:5555
healthcheck:
test: ["CMD", "curl", "--fail", "http://localhost:5555/"]
interval: 10s
timeout: 10s
retries: 5
restart: always
volumes:
postgres-db-volume: