From 0549ea73e17ef729970e7b4467b1ce7f83f759d3 Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Sat, 19 Dec 2020 12:19:10 -0300 Subject: [PATCH 01/20] changing to airflow 2.0.0 --- Dockerfile | 4 +- config/airflow.cfg | 898 +++++++++--------- dags/treino01.py | 41 + dags/treino02.py | 53 ++ dags/treino03.py | 80 ++ dags/treino04.py | 224 +++++ dags/treino05.py | 262 +++++ ...e-CeleryExecutor.yml => docker-compose.yml | 14 +- script/entrypoint.sh | 9 +- 9 files changed, 1128 insertions(+), 457 deletions(-) create mode 100644 dags/treino01.py create mode 100644 dags/treino02.py create mode 100644 dags/treino03.py create mode 100644 dags/treino04.py create mode 100644 dags/treino05.py rename docker-compose-CeleryExecutor.yml => docker-compose.yml (91%) diff --git a/Dockerfile b/Dockerfile index 02782d0c..b15d94d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ # BUILD: docker build --rm -t puckel/docker-airflow . # SOURCE: https://github.com/puckel/docker-airflow -FROM python:3.7-slim-buster +FROM python:3.8-slim-buster LABEL maintainer="Puckel_" # Never prompt the user for choices on installation/configuration of packages @@ -12,7 +12,7 @@ ENV DEBIAN_FRONTEND noninteractive ENV TERM linux # Airflow -ARG AIRFLOW_VERSION=1.10.9 +ARG AIRFLOW_VERSION=2.0.0 ARG AIRFLOW_USER_HOME=/usr/local/airflow ARG AIRFLOW_DEPS="" ARG PYTHON_DEPS="" diff --git a/config/airflow.cfg b/config/airflow.cfg index 9e4d5229..a7dfed82 100644 --- a/config/airflow.cfg +++ b/config/airflow.cfg @@ -3,79 +3,40 @@ # subfolder in a code repository. This path must be absolute. dags_folder = /usr/local/airflow/dags -# The folder where airflow should store its log files -# This path must be absolute -base_log_folder = /usr/local/airflow/logs - -# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search. -# Set this to True if you want to enable remote logging. -remote_logging = False - -# Users must supply an Airflow connection id that provides access to the storage -# location. -remote_log_conn_id = -remote_base_log_folder = -encrypt_s3_logs = False - -# Logging level -logging_level = INFO - -# Logging level for Flask-appbuilder UI -fab_logging_level = WARN - -# Logging class -# Specify the class that will specify the logging configuration -# This class has to be on the python classpath -# Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG -logging_config_class = - -# Flag to enable/disable Colored logs in Console -# Colour the logs when the controlling terminal is a TTY. -colored_console_log = True - -# Log format for when Colored logs is enabled -colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {{%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d}} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s -colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter - -# Format of Log line -log_format = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(levelname)s - %%(message)s -simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s - -# Log filename format -log_filename_template = {{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log -log_processor_filename_template = {{ filename }}.log -dag_processor_manager_log_location = /usr/local/airflow/logs/dag_processor_manager/dag_processor_manager.log - -# Name of handler to read task instance logs. -# Default to use task handler. -task_log_reader = task - # Hostname by providing a path to a callable, which will resolve the hostname. -# The format is "package:function". +# The format is "package.function". # -# For example, default value "socket:getfqdn" means that result from getfqdn() of "socket" +# For example, default value "socket.getfqdn" means that result from getfqdn() of "socket" # package will be used as hostname. # # No argument should be required in the function specified. -# If using IP address as hostname is preferred, use value ``airflow.utils.net:get_host_ip_address`` -hostname_callable = socket:getfqdn +# If using IP address as hostname is preferred, use value ``airflow.utils.net.get_host_ip_address`` +hostname_callable = socket.getfqdn # Default timezone in case supplied date times are naive # can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam) default_timezone = utc # The executor class that airflow should use. Choices include -# SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor, KubernetesExecutor -executor = SequentialExecutor +# ``SequentialExecutor``, ``LocalExecutor``, ``CeleryExecutor``, ``DaskExecutor``, +# ``KubernetesExecutor``, ``CeleryKubernetesExecutor`` or the +# full import path to the class when using a custom executor. +executor = SequentialExecutor # The SqlAlchemy connection string to the metadata database. # SqlAlchemy supports many different database engine, more information # their website -# sql_alchemy_conn = sqlite:////tmp/airflow.db +sql_alchemy_conn = sqlite:////usr/local/airflow/airflow.db # The encoding for the databases sql_engine_encoding = utf-8 +# Collation for ``dag_id``, ``task_id``, ``key`` columns in case they have different encoding. +# This is particularly useful in case of mysql with utf8mb4 encoding because +# primary keys for XCom table has too big size and ``sql_engine_collation_for_ids`` should +# be set to ``utf8mb3_general_ci``. +# sql_engine_collation_for_ids = + # If SqlAlchemy should pool database connections. sql_alchemy_pool_enabled = True @@ -90,8 +51,8 @@ sql_alchemy_pool_size = 5 # It follows then that the total number of simultaneous connections the pool will allow # is pool_size + max_overflow, # and the total number of "sleeping" connections the pool will allow is pool_size. -# max_overflow can be set to -1 to indicate no overflow limit; -# no limit will be placed on the total number of concurrent connections. Defaults to 10. +# max_overflow can be set to ``-1`` to indicate no overflow limit; +# no limit will be placed on the total number of concurrent connections. Defaults to ``10``. sql_alchemy_max_overflow = 10 # The SqlAlchemy pool recycle is the number of seconds a connection @@ -110,12 +71,19 @@ sql_alchemy_pool_pre_ping = True # SqlAlchemy supports databases with the concept of multiple schemas. sql_alchemy_schema = +# Import path for connect args in SqlAlchemy. Defaults to an empty dict. +# This is useful when you want to configure db engine args that SqlAlchemy won't parse +# in connection string. +# See https://docs.sqlalchemy.org/en/13/core/engines.html#sqlalchemy.create_engine.params.connect_args +# sql_alchemy_connect_args = + # The amount of parallelism as a setting to the executor. This defines # the max number of task instances that should run simultaneously # on this airflow installation parallelism = 32 # The number of task instances allowed to run concurrently by the scheduler +# in one DAG. Can be overridden by ``concurrency`` on DAG level. dag_concurrency = 16 # Are DAGs paused by default at creation @@ -124,27 +92,46 @@ dags_are_paused_at_creation = True # The maximum number of active DAG runs per DAG max_active_runs_per_dag = 16 -# Whether to load the examples that ship with Airflow. It's good to -# get started, but you probably want to set this to False in a production +# Whether to load the DAG examples that ship with Airflow. It's good to +# get started, but you probably want to set this to ``False`` in a production # environment load_examples = True -# Where your Airflow plugins are stored +# Whether to load the default connections that ship with Airflow. It's good to +# get started, but you probably want to set this to ``False`` in a production +# environment +load_default_connections = True + +# Path to the folder containing Airflow plugins plugins_folder = /usr/local/airflow/plugins +# Should tasks be executed via forking of the parent process ("False", +# the speedier option) or by spawning a new python process ("True" slow, +# but means plugin changes picked up by tasks straight away) +execute_tasks_new_python_interpreter = False + # Secret key to save connection passwords in the db -fernet_key = $FERNET_KEY +fernet_key = y_xj2l7hU5QKHGn-9o_T9-tIu-pUN1wvXom1ZanIN1w= # Whether to disable pickling dags -donot_pickle = False +donot_pickle = True # How long before timing out a python file import -dagbag_import_timeout = 30 +dagbag_import_timeout = 30.0 + +# Should a traceback be shown in the UI for dagbag import errors, +# instead of just the exception message +dagbag_import_error_tracebacks = True + +# If tracebacks are shown, how many entries from the traceback should be shown +dagbag_import_error_traceback_depth = 2 # How long before timing out a DagFileProcessor, which processes a dag file dag_file_processor_timeout = 50 -# The class to use for running task instances in a subprocess +# The class to use for running task instances in a subprocess. +# Choices include StandardTaskRunner, CgroupTaskRunner or the full import path to the class +# when using a custom task runner. task_runner = StandardTaskRunner # If set, tasks without a ``run_as_user`` argument will be run with this user @@ -154,17 +141,13 @@ default_impersonation = # What security module to use (for example kerberos) security = -# If set to False enables some unsecure features like Charts and Ad Hoc Queries. -# In 2.0 will default to True. -secure_mode = False - # Turn unit test mode on (overwrites many configuration options with test # values at runtime) unit_test_mode = False # Whether to enable pickling for xcom (note that this is insecure and allows for -# RCE exploits). This will be deprecated in Airflow 2.0 (be forced to False). -enable_xcom_pickling = True +# RCE exploits). +enable_xcom_pickling = False # When a task is killed forcefully, this is the amount of time in seconds that # it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED @@ -173,10 +156,7 @@ killed_task_cleanup_time = 60 # Whether to override params with dag_run.conf. If you pass some key-value pairs # through ``airflow dags backfill -c`` or # ``airflow dags trigger -c``, the key-value pairs will override the existing ones in params. -dag_run_conf_overrides_params = False - -# Worker initialisation check to validate Metadata Database connection -worker_precheck = False +dag_run_conf_overrides_params = True # When discovering DAGs, ignore any files that don't contain the strings ``DAG`` and ``airflow``. dag_discovery_safe_mode = True @@ -184,17 +164,166 @@ dag_discovery_safe_mode = True # The number of retries each task is going to have by default. Can be overridden at dag or task level. default_task_retries = 0 -# Whether to serialises DAGs and persist them in DB. -# If set to True, Webserver reads from DB instead of parsing DAG files -# More details: https://airflow.apache.org/docs/stable/dag-serialization.html -store_serialized_dags = False - # Updating serialized DAG can not be faster than a minimum interval to reduce database write rate. min_serialized_dag_update_interval = 30 +# Fetching serialized DAG can not be faster than a minimum interval to reduce database +# read rate. This config controls when your DAGs are updated in the Webserver +min_serialized_dag_fetch_interval = 10 + +# Whether to persist DAG files code in DB. +# If set to True, Webserver reads file contents from DB instead of +# trying to access files in a DAG folder. +# Example: store_dag_code = False +# store_dag_code = + +# Maximum number of Rendered Task Instance Fields (Template Fields) per task to store +# in the Database. +# All the template_fields for each of Task Instance are stored in the Database. +# Keeping this number small may cause an error when you try to view ``Rendered`` tab in +# TaskInstance view for older tasks. +max_num_rendered_ti_fields_per_task = 30 + # On each dagrun check against defined SLAs check_slas = True +# Path to custom XCom class that will be used to store and resolve operators results +# Example: xcom_backend = path.to.CustomXCom +xcom_backend = airflow.models.xcom.BaseXCom + +# By default Airflow plugins are lazily-loaded (only loaded when required). Set it to ``False``, +# if you want to load plugins whenever 'airflow' is invoked via cli or loaded from module. +lazy_load_plugins = True + +# By default Airflow providers are lazily-discovered (discovery and imports happen only when required). +# Set it to False, if you want to discover providers whenever 'airflow' is invoked via cli or +# loaded from module. +lazy_discover_providers = True + +# Number of times the code should be retried in case of DB Operational Errors. +# Not all transactions will be retried as it can cause undesired state. +# Currently it is only used in ``DagFileProcessor.process_file`` to retry ``dagbag.sync_to_db``. +max_db_retries = 3 + +[logging] +# The folder where airflow should store its log files +# This path must be absolute +base_log_folder = /usr/local/airflow/logs + +# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search. +# Set this to True if you want to enable remote logging. +remote_logging = False + +# Users must supply an Airflow connection id that provides access to the storage +# location. +remote_log_conn_id = + +# Path to Google Credential JSON file. If omitted, authorization based on `the Application Default +# Credentials +# `__ will +# be used. +google_key_path = + +# Storage bucket URL for remote logging +# S3 buckets should start with "s3://" +# Cloudwatch log groups should start with "cloudwatch://" +# GCS buckets should start with "gs://" +# WASB buckets should start with "wasb" just to help Airflow select correct handler +# Stackdriver logs should start with "stackdriver://" +remote_base_log_folder = + +# Use server-side encryption for logs stored in S3 +encrypt_s3_logs = False + +# Logging level +logging_level = INFO + +# Logging level for Flask-appbuilder UI +fab_logging_level = WARN + +# Logging class +# Specify the class that will specify the logging configuration +# This class has to be on the python classpath +# Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG +logging_config_class = + +# Flag to enable/disable Colored logs in Console +# Colour the logs when the controlling terminal is a TTY. +colored_console_log = True + +# Log format for when Colored logs is enabled +colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s +colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter + +# Format of Log line +log_format = [%%(asctime)s] {%%(filename)s:%%(lineno)d} %%(levelname)s - %%(message)s +simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s + +# Specify prefix pattern like mentioned below with stream handler TaskHandlerWithCustomFormatter +# Example: task_log_prefix_template = {ti.dag_id}-{ti.task_id}-{execution_date}-{try_number} +task_log_prefix_template = + +# Formatting for how airflow generates file names/paths for each task run. +log_filename_template = {{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log + +# Formatting for how airflow generates file names for log +log_processor_filename_template = {{ filename }}.log + +# full path of dag_processor_manager logfile +dag_processor_manager_log_location = /usr/local/airflow/logs/dag_processor_manager/dag_processor_manager.log + +# Name of handler to read task instance logs. +# Defaults to use ``task`` handler. +task_log_reader = task + +# A comma\-separated list of third-party logger names that will be configured to print messages to +# consoles\. +# Example: extra_loggers = connexion,sqlalchemy +extra_loggers = + +[metrics] + +# StatsD (https://github.com/etsy/statsd) integration settings. +# Enables sending metrics to StatsD. +statsd_on = False +statsd_host = localhost +statsd_port = 8125 +statsd_prefix = airflow + +# If you want to avoid sending all the available metrics to StatsD, +# you can configure an allow list of prefixes (comma separated) to send only the metrics that +# start with the elements of the list (e.g: "scheduler,executor,dagrun") +statsd_allow_list = + +# A function that validate the statsd stat name, apply changes to the stat name if necessary and return +# the transformed stat name. +# +# The function should have the following signature: +# def func_name(stat_name: str) -> str: +stat_name_handler = + +# To enable datadog integration to send airflow metrics. +statsd_datadog_enabled = False + +# List of datadog tags attached to all metrics(e.g: key1:value1,key2:value2) +statsd_datadog_tags = + +# If you want to utilise your own custom Statsd client set the relevant +# module path below. +# Note: The module path must exist on your PYTHONPATH for Airflow to pick it up +# statsd_custom_client_path = + +[secrets] +# Full class name of secrets backend to enable (will precede env vars and metastore in search path) +# Example: backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend +backend = + +# The backend_kwargs param is loaded into a dictionary and passed to __init__ of secrets backend class. +# See documentation for the secrets backend you are using. JSON is expected. +# Example for AWS Systems Manager ParameterStore: +# ``{"connections_prefix": "/airflow/connections", "profile_name": "default"}`` +backend_kwargs = + [cli] # In what way should the cli access the API. The LocalClient will use the # database directly, while the json_client will use the api running on the @@ -207,13 +336,47 @@ api_client = airflow.api.client.local_client endpoint_url = http://localhost:8080 [debug] -# Used only with DebugExecutor. If set to True DAG will fail with first +# Used only with ``DebugExecutor``. If set to ``True`` DAG will fail with first # failed task. Helpful for debugging purposes. fail_fast = False [api] -# How to authenticate users of the API -auth_backend = airflow.api.auth.backend.default +# Enables the deprecated experimental API. Please note that these APIs do not have access control. +# The authenticated user has full access. +# +# .. warning:: +# +# This `Experimental REST API `__ is +# deprecated since version 2.0. Please consider using +# `the Stable REST API `__. +# For more information on migration, see +# `UPDATING.md `_ +enable_experimental_api = False + +# How to authenticate users of the API. See +# https://airflow.apache.org/docs/stable/security.html for possible values. +# ("airflow.api.auth.backend.default" allows all requests for historic reasons) +auth_backend = airflow.api.auth.backend.deny_all + +# Used to set the maximum page limit for API requests +maximum_page_limit = 100 + +# Used to set the default page limit when limit is zero. A default limit +# of 100 is set on OpenApi spec. However, this particular default limit +# only work when limit is set equal to zero(0) from API requests. +# If no limit is supplied, the OpenApi spec default is used. +fallback_page_limit = 100 + +# The intended audience for JWT token credentials used for authorization. This value must match on the client and server sides. If empty, audience will not be tested. +# Example: google_oauth2_audience = project-id-random-value.apps.googleusercontent.com +google_oauth2_audience = + +# Path to Google Cloud Service Account key file (JSON). If omitted, authorization based on +# `the Application Default Credentials +# `__ will +# be used. +# Example: google_key_path = /files/service-account-json +google_key_path = [lineage] # what lineage backend to use @@ -235,16 +398,30 @@ default_ram = 512 default_disk = 512 default_gpus = 0 +# Is allowed to pass additional/unused arguments (args, kwargs) to the BaseOperator operator. +# If set to False, an exception will be thrown, otherwise only the console message will be displayed. +allow_illegal_arguments = False + [hive] # Default mapreduce queue for HiveOperator tasks default_hive_mapred_queue = +# Template for mapred_job_name in HiveOperator, supports the following named parameters +# hostname, dag_id, task_id, execution_date +# mapred_job_name_template = + [webserver] # The base url of your website as airflow cannot guess what domain or # cname you are using. This is used in automated emails that # airflow sends to point links to the right web server base_url = http://localhost:8080 +# Default timezone to display all dates in the UI, can be UTC, system, or +# any IANA timezone string (e.g. Europe/Amsterdam). If left empty the +# default value of core/default_timezone will be used +# Example: default_ui_timezone = America/New_York +default_ui_timezone = UTC + # The ip specified when starting the web server web_server_host = 0.0.0.0 @@ -273,9 +450,13 @@ worker_refresh_batch_size = 1 # Number of seconds to wait before refreshing a batch of workers. worker_refresh_interval = 30 +# If set to True, Airflow will track files in plugins_folder directory. When it detects changes, +# then reload the gunicorn. +reload_on_plugin_change = False + # Secret key used to run your flask app # It should be as random as possible -secret_key = temporary_key +secret_key = 5b5GOkm5PddhiGyFsoEZFw== # Number of workers to run the Gunicorn web server workers = 4 @@ -290,8 +471,13 @@ access_logfile = - # Log files for the gunicorn webserver. '-' means log to stderr. error_logfile = - +# Access log format for gunicorn webserver. +# default format is %%(h)s %%(l)s %%(u)s %%(t)s "%%(r)s" %%(s)s %%(b)s "%%(f)s" "%%(a)s" +# documentation - https://docs.gunicorn.org/en/stable/settings.html#access-log-format +access_logformat = + # Expose the configuration file in the web server -expose_config = True +expose_config = False # Expose hostname in the web server expose_hostname = True @@ -299,26 +485,11 @@ expose_hostname = True # Expose stacktrace in the web server expose_stacktrace = True -# Set to true to turn on authentication: -# https://airflow.apache.org/security.html#web-authentication -authenticate = False - -# Filter the list of dags by owner name (requires authentication to be enabled) -filter_by_owner = False - -# Filtering mode. Choices include user (default) and ldapgroup. -# Ldap group filtering requires using the ldap backend -# -# Note that the ldap server needs the "memberOf" overlay to be set up -# in order to user the ldapgroup mode. -owner_mode = user - -# Default DAG view. Valid values are: -# tree, graph, duration, gantt, landing_times +# Default DAG view. Valid values are: ``tree``, ``graph``, ``duration``, ``gantt``, ``landing_times`` dag_default_view = tree -# "Default DAG orientation. Valid values are:" -# LR (Left->Right), TB (Top->Bottom), RL (Right->Left), BT (Bottom->Top) +# Default DAG orientation. Valid values are: +# ``LR`` (Left->Right), ``TB`` (Top->Bottom), ``RL`` (Right->Left), ``BT`` (Bottom->Top) dag_orientation = LR # Puts the webserver in demonstration mode; blurs the names of Operators for @@ -345,11 +516,8 @@ hide_paused_dags_by_default = False # Consistent page size across all listing views in the UI page_size = 100 -# Use FAB-based webserver with RBAC feature -rbac = False - # Define the color of navigation bar -navbar_color = #007A87 +navbar_color = #fff # Default dagrun to show in UI default_dag_run_display_number = 25 @@ -377,7 +545,7 @@ proxy_fix_x_prefix = 1 cookie_secure = False # Set samesite policy on session cookie -cookie_samesite = +cookie_samesite = Lax # Default setting for wrap toggle on DAG code and TI log views. default_wrap = False @@ -392,20 +560,30 @@ x_frame_enabled = True # Unique ID of your account in the analytics tool # analytics_id = +# 'Recent Tasks' stats will show for old DagRuns if set +show_recent_stats_for_completed_runs = True + # Update FAB permissions and sync security manager roles # on webserver startup update_fab_perms = True -# Minutes of non-activity before logged out from UI -# 0 means never get forcibly logged out -force_log_out_after = 0 - -# The UI cookie lifetime in days -session_lifetime_days = 30 +# The UI cookie lifetime in minutes. User will be logged out from UI after +# ``session_lifetime_minutes`` of non-activity +session_lifetime_minutes = 43200 [email] + +# Configuration email backend and whether to +# send email alerts on retry or failure +# Email backend to use email_backend = airflow.utils.email.send_email_smtp +# Whether email alerts should be sent when a task is retried +default_email_on_retry = True + +# Whether email alerts should be sent when a task failed +default_email_on_failure = True + [smtp] # If you want airflow to send emails on retries, failure, and you want to use @@ -420,12 +598,29 @@ smtp_ssl = False # smtp_password = smtp_port = 25 smtp_mail_from = airflow@example.com +smtp_timeout = 30 +smtp_retry_limit = 5 [sentry] -# Sentry (https://docs.sentry.io) integration +# Sentry (https://docs.sentry.io) integration. Here you can supply +# additional configuration options based on the Python platform. See: +# https://docs.sentry.io/error-reporting/configuration/?platform=python. +# Unsupported options: ``integrations``, ``in_app_include``, ``in_app_exclude``, +# ``ignore_errors``, ``before_breadcrumb``, ``before_send``, ``transport``. +# Enable error reporting to Sentry +sentry_on = false sentry_dsn = +[celery_kubernetes_executor] + +# This section only applies if you are using the ``CeleryKubernetesExecutor`` in +# ``[core]`` section above +# Define when to send a task to ``KubernetesExecutor`` when using ``CeleryKubernetesExecutor``. +# When the queue of a task is ``kubernetes_queue``, the task is executed via ``KubernetesExecutor``, +# otherwise via ``CeleryExecutor`` +kubernetes_queue = kubernetes + [celery] # This section only applies if you are using the CeleryExecutor in @@ -437,7 +632,7 @@ celery_app_name = airflow.executors.celery_executor # ``airflow celery worker`` command. This defines the number of task instances that # a worker will take, so size up your workers based on the resources on # your worker box and the nature of your tasks -worker_concurrency = 16 +worker_concurrency = 8 # The maximum and minimum concurrency that will be used when starting workers with the # ``airflow celery worker`` command (always keep minimum processes, but grow @@ -446,7 +641,17 @@ worker_concurrency = 16 # If autoscale option is available, worker_concurrency will be ignored. # http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale # Example: worker_autoscale = 16,12 -worker_autoscale = 16,12 +# worker_autoscale = + +# Used to increase the number of tasks that a worker prefetches which can improve performance. +# The number of processes multiplied by worker_prefetch_multiplier is the number of tasks +# that are prefetched by a worker. A value greater than 1 can result in tasks being unnecessarily +# blocked if there are multiple workers and one worker prefetches tasks that sit behind long +# running tasks while another worker has unutilized processes that are unable to process the already +# claimed blocked tasks. +# https://docs.celeryproject.org/en/stable/userguide/optimizing.html#prefetch-limits +# Example: worker_prefetch_multiplier = 1 +# worker_prefetch_multiplier = # When you start an airflow worker, airflow starts a tiny web server # subprocess to serve the workers local log files to the airflow main @@ -455,11 +660,14 @@ worker_autoscale = 16,12 # visible from the main web server to connect into the workers. worker_log_server_port = 8793 +# Umask that will be used when starting workers with the ``airflow celery worker`` +# in daemon mode. This control the file-creation mode mask which determines the initial +# value of file permission bits for newly created files. +worker_umask = 0o077 + # The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally -# a sqlalchemy database. Refer to the Celery documentation for more -# information. -# http://docs.celeryproject.org/en/latest/userguide/configuration.html#broker-settings -broker_url = redis://redis:6379/1 +# a sqlalchemy database. Refer to the Celery documentation for more information. +broker_url = redis://redis:6379/0 # The Celery result_backend. When a job finishes, it needs to update the # metadata of the job. Therefore it will post a message on a message bus, @@ -467,10 +675,10 @@ broker_url = redis://redis:6379/1 # This status is used by the scheduler to update the state of the task # The use of a database is highly recommended # http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings -result_backend = db+postgresql://airflow:airflow@postgres/airflow +result_backend = db+postgresql://postgres:airflow@postgres/airflow # Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start -# it ``airflow flower``. This defines the IP that Celery Flower runs on +# it ``airflow celery flower``. This defines the IP that Celery Flower runs on flower_host = 0.0.0.0 # The root URL for Flower @@ -494,15 +702,13 @@ sync_parallelism = 0 # Import path for celery configuration options celery_config_options = airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG - -# In case of using SSL ssl_active = False ssl_key = ssl_cert = ssl_cacert = # Celery Pool implementation. -# Choices include: prefork (default), eventlet, gevent or solo. +# Choices include: ``prefork`` (default), ``eventlet``, ``gevent`` or ``solo``. # See: # https://docs.celeryproject.org/en/latest/userguide/workers.html#concurrency # https://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html @@ -510,7 +716,23 @@ pool = prefork # The number of seconds to wait before timing out ``send_task_to_executor`` or # ``fetch_celery_task_state`` operations. -operation_timeout = 2 +operation_timeout = 1.0 + +# Celery task will report its status as 'started' when the task is executed by a worker. +# This is used in Airflow to keep track of the running tasks and if a Scheduler is restarted +# or run in HA mode, it can adopt the orphan tasks launched by previous SchedulerJob. +task_track_started = True + +# Time in seconds after which Adopted tasks are cleared by CeleryExecutor. This is helpful to clear +# stalled tasks. +task_adoption_timeout = 600 + +# The Maximum number of retries for publishing task messages to the broker when failing +# due to ``AirflowTaskTimeout`` error before giving up and marking Task as failed. +task_publish_max_retries = 3 + +# Worker initialisation check to validate Metadata Database connection +worker_precheck = False [celery_broker_transport_options] @@ -545,15 +767,15 @@ tls_key = # listen (in seconds). job_heartbeat_sec = 5 +# How often (in seconds) to check and tidy up 'running' TaskInstancess +# that no longer have a matching DagRun +clean_tis_without_dagrun_interval = 15.0 + # The scheduler constantly tries to trigger new tasks (look at the # scheduler section in the docs for more information). This defines # how often the scheduler should run (in seconds). scheduler_heartbeat_sec = 5 -# After how much time should the scheduler terminate in seconds -# -1 indicates to run continuously (see also num_runs) -run_duration = -1 - # The number of times to try to schedule each DAG file # -1 indicates unlimited number num_runs = -1 @@ -570,10 +792,16 @@ dag_dir_list_interval = 300 # How often should stats be printed to the logs. Setting to 0 will disable printing stats print_stats_interval = 30 +# How often (in seconds) should pool usage stats be sent to statsd (if statsd_on is enabled) +pool_metrics_interval = 5.0 + # If the last scheduler heartbeat happened more than scheduler_health_check_threshold # ago (in seconds), scheduler is considered unhealthy. # This is used by the health check in the "/health" endpoint scheduler_health_check_threshold = 30 + +# How often (in seconds) should the scheduler check for orphaned tasks and SchedulerJobs +orphaned_tasks_check_interval = 300.0 child_process_log_directory = /usr/local/airflow/logs/scheduler # Local task jobs periodically heartbeat to the DB. If the job has @@ -581,10 +809,10 @@ child_process_log_directory = /usr/local/airflow/logs/scheduler # associated task instance as failed and will re-schedule the task. scheduler_zombie_task_threshold = 300 -# Turn off scheduler catchup by setting this to False. +# Turn off scheduler catchup by setting this to ``False``. # Default behavior is unchanged and # Command Line Backfills still work, but the scheduler -# will not do scheduler catchup if this is False, +# will not do scheduler catchup if this is ``False``, # however it can be set on a per DAG basis in the # DAG definition (catchup) catchup_by_default = True @@ -599,21 +827,32 @@ catchup_by_default = True # Set this to 0 for no limit (not advised) max_tis_per_query = 512 -# Statsd (https://github.com/etsy/statsd) integration settings -statsd_on = False -statsd_host = localhost -statsd_port = 8125 -statsd_prefix = airflow +# Should the scheduler issue ``SELECT ... FOR UPDATE`` in relevant queries. +# If this is set to False then you should not run more than a single +# scheduler at once +use_row_level_locking = True -# If you want to avoid send all the available metrics to StatsD, -# you can configure an allow list of prefixes to send only the metrics that -# start with the elements of the list (e.g: scheduler,executor,dagrun) -statsd_allow_list = +# Max number of DAGs to create DagRuns for per scheduler loop +# +# Default: 10 +# max_dagruns_to_create_per_loop = + +# How many DagRuns should a scheduler examine (and lock) when scheduling +# and queuing tasks. +# +# Default: 20 +# max_dagruns_per_loop_to_schedule = + +# Should the Task supervisor process perform a "mini scheduler" to attempt to schedule more tasks of the +# same DAG. Leaving this on will mean tasks in the same DAG execute quicker, but might starve out other +# dags in some circumstances +# +# Default: True +# schedule_after_task_execution = -# The scheduler can run multiple threads in parallel to schedule dags. -# This defines how many threads will run. -max_threads = 2 -authenticate = False +# The scheduler can run multiple processes in parallel to parse dags. +# This defines how many processes will run. +parsing_processes = 2 # Turn off scheduler use of cron intervals by setting this to False. # DAGs submitted manually in the web UI or with trigger_dag will still run. @@ -623,70 +862,6 @@ use_job_schedule = True # Only has effect if schedule_interval is set to None in DAG allow_trigger_in_future = False -[ldap] -# set this to ldaps://: -uri = -user_filter = objectClass=* -user_name_attr = uid -group_member_attr = memberOf -superuser_filter = -data_profiler_filter = -bind_user = cn=Manager,dc=example,dc=com -bind_password = insecure -basedn = dc=example,dc=com -cacert = /etc/ca/ldap_ca.crt -search_scope = LEVEL - -# This setting allows the use of LDAP servers that either return a -# broken schema, or do not return a schema. -ignore_malformed_schema = False - -[mesos] -# Mesos master address which MesosExecutor will connect to. -master = localhost:5050 - -# The framework name which Airflow scheduler will register itself as on mesos -framework_name = Airflow - -# Number of cpu cores required for running one task instance using -# 'airflow run --local -p ' -# command on a mesos slave -task_cpu = 1 - -# Memory in MB required for running one task instance using -# 'airflow run --local -p ' -# command on a mesos slave -task_memory = 256 - -# Enable framework checkpointing for mesos -# See http://mesos.apache.org/documentation/latest/slave-recovery/ -checkpoint = False - -# Failover timeout in milliseconds. -# When checkpointing is enabled and this option is set, Mesos waits -# until the configured timeout for -# the MesosExecutor framework to re-register after a failover. Mesos -# shuts down running tasks if the -# MesosExecutor framework fails to re-register within this timeframe. -# Example: failover_timeout = 604800 -# failover_timeout = - -# Enable framework authentication for mesos -# See http://mesos.apache.org/documentation/latest/configuration/ -authenticate = False - -# Mesos credentials, if authentication is enabled -# Example: default_principal = admin -# default_principal = -# Example: default_secret = admin -# default_secret = - -# Optional Docker Image to run on slave before running the command -# This image should be accessible from mesos slave i.e mesos slave -# should be able to pull this docker image before executing the command. -# Example: docker_image_slave = puckel/docker-airflow -# docker_image_slave = - [kerberos] ccache = /tmp/airflow_krb5_ccache @@ -703,12 +878,15 @@ api_rev = v3 # UI to hide sensitive variable fields when set to True hide_sensitive_variable_fields = True +# A comma-separated list of sensitive keywords to look for in variables names. +sensitive_variable_fields = + [elasticsearch] # Elasticsearch host host = # Format of the log_id, which is used to query for a given tasks logs -log_id_template = {{dag_id}}-{{task_id}}-{{execution_date}}-{{try_number}} +log_id_template = {dag_id}-{task_id}-{execution_date}-{try_number} # Used to mark the end of a log stream for a task end_of_log_mark = end_of_log @@ -732,175 +910,35 @@ use_ssl = False verify_certs = True [kubernetes] -# The repository, tag and imagePullPolicy of the Kubernetes Image for the Worker to Run -worker_container_repository = -worker_container_tag = -worker_container_image_pull_policy = IfNotPresent +# Path to the YAML pod file. If set, all other kubernetes-related fields are ignored. +pod_template_file = -# If True (default), worker pods will be deleted upon termination -delete_worker_pods = True +# The repository of the Kubernetes Image for the Worker to Run +worker_container_repository = -# Number of Kubernetes Worker Pod creation calls per scheduler loop -worker_pods_creation_batch_size = 1 +# The tag of the Kubernetes Image for the Worker to Run +worker_container_tag = # The Kubernetes namespace where airflow workers should be created. Defaults to ``default`` namespace = default -# The name of the Kubernetes ConfigMap containing the Airflow Configuration (this file) -# Example: airflow_configmap = airflow-configmap -airflow_configmap = +# If True, all worker pods will be deleted upon termination +delete_worker_pods = True -# The name of the Kubernetes ConfigMap containing ``airflow_local_settings.py`` file. -# -# For example: -# -# ``airflow_local_settings_configmap = "airflow-configmap"`` if you have the following ConfigMap. -# -# ``airflow-configmap.yaml``: -# -# .. code-block:: yaml -# -# --- -# apiVersion: v1 -# kind: ConfigMap -# metadata: -# name: airflow-configmap -# data: -# airflow_local_settings.py: | -# def pod_mutation_hook(pod): -# ... -# airflow.cfg: | -# ... -# Example: airflow_local_settings_configmap = airflow-configmap -airflow_local_settings_configmap = - -# For docker image already contains DAGs, this is set to ``True``, and the worker will -# search for dags in dags_folder, -# otherwise use git sync or dags volume claim to mount DAGs -dags_in_image = False - -# For either git sync or volume mounted DAGs, the worker will look in this subpath for DAGs -dags_volume_subpath = - -# For DAGs mounted via a volume claim (mutually exclusive with git-sync and host path) -dags_volume_claim = - -# For volume mounted logs, the worker will look in this subpath for logs -logs_volume_subpath = - -# A shared volume claim for the logs -logs_volume_claim = - -# For DAGs mounted via a hostPath volume (mutually exclusive with volume claim and git-sync) -# Useful in local environment, discouraged in production -dags_volume_host = - -# A hostPath volume for the logs -# Useful in local environment, discouraged in production -logs_volume_host = - -# A list of configMapsRefs to envFrom. If more than one configMap is -# specified, provide a comma separated list: configmap_a,configmap_b -env_from_configmap_ref = - -# A list of secretRefs to envFrom. If more than one secret is -# specified, provide a comma separated list: secret_a,secret_b -env_from_secret_ref = - -# Git credentials and repository for DAGs mounted via Git (mutually exclusive with volume claim) -git_repo = -git_branch = -git_subpath = - -# The specific rev or hash the git_sync init container will checkout -# This becomes GIT_SYNC_REV environment variable in the git_sync init container for worker pods -git_sync_rev = - -# Use git_user and git_password for user authentication or git_ssh_key_secret_name -# and git_ssh_key_secret_key for SSH authentication -git_user = -git_password = -git_sync_root = /git -git_sync_dest = repo - -# Mount point of the volume if git-sync is being used. -# i.e. /usr/local/airflow/dags -git_dags_folder_mount_point = - -# To get Git-sync SSH authentication set up follow this format -# -# ``airflow-secrets.yaml``: -# -# .. code-block:: yaml -# -# --- -# apiVersion: v1 -# kind: Secret -# metadata: -# name: airflow-secrets -# data: -# # key needs to be gitSshKey -# gitSshKey: -# Example: git_ssh_key_secret_name = airflow-secrets -git_ssh_key_secret_name = - -# To get Git-sync SSH authentication set up follow this format -# -# ``airflow-configmap.yaml``: -# -# .. code-block:: yaml -# -# --- -# apiVersion: v1 -# kind: ConfigMap -# metadata: -# name: airflow-configmap -# data: -# known_hosts: | -# github.com ssh-rsa <...> -# airflow.cfg: | -# ... -# Example: git_ssh_known_hosts_configmap_name = airflow-configmap -git_ssh_known_hosts_configmap_name = - -# To give the git_sync init container credentials via a secret, create a secret -# with two fields: GIT_SYNC_USERNAME and GIT_SYNC_PASSWORD (example below) and -# add ``git_sync_credentials_secret = `` to your airflow config under the -# ``kubernetes`` section -# -# Secret Example: -# -# .. code-block:: yaml -# -# --- -# apiVersion: v1 -# kind: Secret -# metadata: -# name: git-credentials -# data: -# GIT_SYNC_USERNAME: -# GIT_SYNC_PASSWORD: -git_sync_credentials_secret = - -# For cloning DAGs from git repositories into volumes: https://github.com/kubernetes/git-sync -git_sync_container_repository = k8s.gcr.io/git-sync -git_sync_container_tag = v3.1.1 -git_sync_init_container_name = git-sync-clone -git_sync_run_as_user = 65533 - -# The name of the Kubernetes service account to be associated with airflow workers, if any. -# Service accounts are required for workers that require access to secrets or cluster resources. -# See the Kubernetes RBAC documentation for more: -# https://kubernetes.io/docs/admin/authorization/rbac/ -worker_service_account_name = - -# Any image pull secrets to be given to worker pods, If more than one secret is -# required, provide a comma separated list: secret_a,secret_b -image_pull_secrets = - -# GCP Service Account Keys to be provided to tasks run on Kubernetes Executors -# Should be supplied in the format: key-name-1:key-path-1,key-name-2:key-path-2 -gcp_service_account_keys = +# If False (and delete_worker_pods is True), +# failed worker pods will not be deleted so users can investigate them. +delete_worker_pods_on_failure = False + +# Number of Kubernetes Worker Pod creation calls per scheduler loop. +# Note that the current default of "1" will only launch a single pod +# per-heartbeat. It is HIGHLY recommended that users increase this +# number to match the tolerance of their kubernetes cluster for +# better performance. +worker_pods_creation_batch_size = 1 + +# Allows users to launch pods in multiple namespaces. +# Will require creating a cluster-role for the scheduler +multi_namespace_mode = False # Use the service account kubernetes gives to pods to connect to kubernetes cluster. # It's intended for clients that expect to be running inside a pod running on kubernetes. @@ -910,81 +948,53 @@ in_cluster = True # When running with in_cluster=False change the default cluster_context or config_file # options to Kubernetes client. Leave blank these to use default behaviour like ``kubectl`` has. # cluster_context = -# config_file = -# Affinity configuration as a single line formatted JSON object. -# See the affinity model for top-level key names (e.g. ``nodeAffinity``, etc.): -# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#affinity-v1-core -affinity = - -# A list of toleration objects as a single line formatted JSON array -# See: -# https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.12/#toleration-v1-core -tolerations = +# Path to the kubernetes configfile to be used when ``in_cluster`` is set to False +# config_file = # Keyword parameters to pass while calling a kubernetes client core_v1_api methods # from Kubernetes Executor provided as a single line formatted JSON dictionary string. # List of supported params are similar for all core_v1_apis, hence a single config -# variable for all apis. -# See: -# https://raw.githubusercontent.com/kubernetes-client/python/master/kubernetes/client/apis/core_v1_api.py -# Note that if no _request_timeout is specified, the kubernetes client will wait indefinitely -# for kubernetes api responses, which will cause the scheduler to hang. -# The timeout is specified as [connect timeout, read timeout] -kube_client_request_args = {{"_request_timeout" : [60,60] }} - -# Specifies the uid to run the first process of the worker pods containers as -run_as_user = - -# Specifies a gid to associate with all containers in the worker pods -# if using a git_ssh_key_secret_name use an fs_group -# that allows for the key to be read, e.g. 65533 -fs_group = - -[kubernetes_node_selectors] - -# The Key-value pairs to be given to worker pods. -# The worker pods will be scheduled to the nodes of the specified key-value pairs. -# Should be supplied in the format: key = value - -[kubernetes_annotations] - -# The Key-value annotations pairs to be given to worker pods. -# Should be supplied in the format: key = value - -[kubernetes_environment_variables] - -# The scheduler sets the following environment variables into your workers. You may define as -# many environment variables as needed and the kubernetes launcher will set them in the launched workers. -# Environment variables in this section are defined as follows -# `` = `` -# -# For example if you wanted to set an environment variable with value `prod` and key -# ``ENVIRONMENT`` you would follow the following format: -# ENVIRONMENT = prod -# -# Additionally you may override worker airflow settings with the ``AIRFLOW__
__`` -# formatting as supported by airflow normally. - -[kubernetes_secrets] - -# The scheduler mounts the following secrets into your workers as they are launched by the -# scheduler. You may define as many secrets as needed and the kubernetes launcher will parse the -# defined secrets and mount them as secret environment variables in the launched workers. -# Secrets in this section are defined as follows -# `` = =`` -# -# For example if you wanted to mount a kubernetes secret key named ``postgres_password`` from the -# kubernetes secret object ``airflow-secret`` as the environment variable ``POSTGRES_PASSWORD`` into -# your workers you would follow the following format: -# ``POSTGRES_PASSWORD = airflow-secret=postgres_credentials`` -# -# Additionally you may override worker airflow settings with the ``AIRFLOW__
__`` -# formatting as supported by airflow normally. - -[kubernetes_labels] - -# The Key-value pairs to be given to worker pods. -# The worker pods will be given these static labels, as well as some additional dynamic labels -# to identify the task. -# Should be supplied in the format: ``key = value`` +# variable for all apis. See: +# https://raw.githubusercontent.com/kubernetes-client/python/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/api/core_v1_api.py +kube_client_request_args = + +# Optional keyword arguments to pass to the ``delete_namespaced_pod`` kubernetes client +# ``core_v1_api`` method when using the Kubernetes Executor. +# This should be an object and can contain any of the options listed in the ``v1DeleteOptions`` +# class defined here: +# https://github.com/kubernetes-client/python/blob/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/models/v1_delete_options.py#L19 +# Example: delete_option_kwargs = {"grace_period_seconds": 10} +delete_option_kwargs = + +# Enables TCP keepalive mechanism. This prevents Kubernetes API requests to hang indefinitely +# when idle connection is time-outed on services like cloud load balancers or firewalls. +enable_tcp_keepalive = False + +# When the `enable_tcp_keepalive` option is enabled, TCP probes a connection that has +# been idle for `tcp_keep_idle` seconds. +tcp_keep_idle = 120 + +# When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond +# to a keepalive probe, TCP retransmits the probe after `tcp_keep_intvl` seconds. +tcp_keep_intvl = 30 + +# When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond +# to a keepalive probe, TCP retransmits the probe `tcp_keep_cnt number` of times before +# a connection is considered to be broken. +tcp_keep_cnt = 6 + +[smart_sensor] +# When `use_smart_sensor` is True, Airflow redirects multiple qualified sensor tasks to +# smart sensor task. +use_smart_sensor = False + +# `shard_code_upper_limit` is the upper limit of `shard_code` value. The `shard_code` is generated +# by `hashcode % shard_code_upper_limit`. +shard_code_upper_limit = 10000 + +# The number of running smart sensor processes for each service. +shards = 5 + +# comma separated sensor classes support in smart_sensor. +sensors_enabled = NamedHivePartitionSensor diff --git a/dags/treino01.py b/dags/treino01.py new file mode 100644 index 00000000..98c1e974 --- /dev/null +++ b/dags/treino01.py @@ -0,0 +1,41 @@ +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.python_operator import PythonOperator +from datetime import datetime, timedelta + +default_args = { + 'owner': 'Neylson Crepalde', + "depends_on_past": False, + "start_date": datetime(2020, 11, 14), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=2), +} + +dag = DAG( + "treino-01", + description="Básico de Bash operators e Python operators", + default_args=default_args, + schedule_interval=timedelta(minutes=1) +) + +def hello_word(): + print("Hello Airflow from Python") + + +hello_bash = BashOperator( + task_id="hello-bash", + bash_command='echo "Hello Airflow from bash"', + dag=dag +) + +hello_python = PythonOperator( + task_id='hello-python', + python_callable=hello_word, + dag=dag +) + + +hello_bash >> hello_python diff --git a/dags/treino02.py b/dags/treino02.py new file mode 100644 index 00000000..a8903b8d --- /dev/null +++ b/dags/treino02.py @@ -0,0 +1,53 @@ +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.python_operator import PythonOperator +from datetime import datetime, timedelta +import pandas as pd + +default_args = { + 'owner': 'Neylson Crepalde', + "depends_on_past": False, + "start_date": datetime(2020, 11, 14, 23, 50), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=1), +} + +dag = DAG( + "treino-02", + description="Get Titanic data from internet and calculate mean age", + default_args=default_args, + schedule_interval=timedelta(minutes=5) +) + +get_data = BashOperator( + task_id="get-data", + bash_command='curl https://raw.githubusercontent.com/A3Data/hermione/master/hermione/file_text/train.csv -o ~/train.csv', + dag=dag +) + +def calculate_mean_age(): + df = pd.read_csv('~/train.csv') + med = df.Age.mean() + return med + +task_calculate_mean = PythonOperator( + task_id='calculate-mean-age', + python_callable=calculate_mean_age, + dag=dag +) + +def print_age(**context): + value = context['task_instance'].xcom_pull(task_ids='calculate-mean-age') + print(f"Mean age is {value}") + +task_mean_age = PythonOperator( + task_id="say-mean-age", + python_callable=print_age, + provide_context=True, + dag=dag +) + +get_data >> task_calculate_mean >> task_mean_age \ No newline at end of file diff --git a/dags/treino03.py b/dags/treino03.py new file mode 100644 index 00000000..5fa19f9a --- /dev/null +++ b/dags/treino03.py @@ -0,0 +1,80 @@ +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.python_operator import PythonOperator, BranchPythonOperator +from datetime import datetime, timedelta +import pandas as pd +import random + +default_args = { + 'owner': 'Neylson Crepalde', + "depends_on_past": False, + "start_date": datetime(2020, 11, 15, 16, 20), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=1), +} + +dag = DAG( + "treino-03", + description="Pega dados do Titanic e calcula idade média para homens ou mulheres", + default_args=default_args, + schedule_interval=timedelta(minutes=2) +) + +get_data = BashOperator( + task_id="get-data", + bash_command='curl https://raw.githubusercontent.com/A3Data/hermione/master/hermione/file_text/train.csv -o /usr/local/airflow/data/train.csv', + dag=dag +) + + +def sorteia_h_m(): + return random.choice(['male', 'female']) + +escolhe_h_m = PythonOperator( + task_id='escolhe-h-m', + python_callable=sorteia_h_m, + dag=dag +) + +def MouF(**context): + value = context['task_instance'].xcom_pull(task_ids='escolhe-h-m') + if value == 'male': + return 'branch_homem' + if value == 'female': + return 'branch_mulher' + +male_female = BranchPythonOperator( + task_id='condicional', + python_callable=MouF, + provide_context=True, + dag=dag +) + + +def mean_homen(): + df = pd.read_csv('/usr/local/airflow/data/train.csv') + df = df.loc[df.Sex == 'male'] + print(f"Média de idade dos homens no Titanic: {df.Age.mean()}") + +branch_homem = PythonOperator( + task_id='branch_homem', + python_callable=mean_homen, + dag=dag +) + +def mean_mulher(): + df = pd.read_csv('/usr/local/airflow/data/train.csv') + df = df.loc[df.Sex == 'female'] + print(f"Média de idade das mulheres no Titanic: {df.Age.mean()}") + +branch_mulher = PythonOperator( + task_id='branch_mulher', + python_callable=mean_mulher, + dag=dag +) + + +get_data >> escolhe_h_m >> male_female >> [branch_homem, branch_mulher] diff --git a/dags/treino04.py b/dags/treino04.py new file mode 100644 index 00000000..f2dd834c --- /dev/null +++ b/dags/treino04.py @@ -0,0 +1,224 @@ +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.python_operator import PythonOperator, BranchPythonOperator +from datetime import datetime, timedelta +import pandas as pd +import zipfile + +# Constantes +data_path = '/usr/local/airflow/data/microdados_enade_2019/2019/3.DADOS/' +arquivo = data_path + 'microdados_enade_2019.txt' + +default_args = { + 'owner': 'Neylson Crepalde', + "depends_on_past": False, + "start_date": datetime(2020, 11, 15, 17), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=1), +} + +dag = DAG( + "treino-04", + description="Paralelismos", + default_args=default_args, + schedule_interval="*/10 * * * *" +) + +start_preprocessing = BashOperator( + task_id='start_preprocessing', + bash_command='echo "Start Preprocessing! Vai!"', + dag=dag +) + +get_data = BashOperator( + task_id="get-data", + bash_command='curl http://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', + dag=dag +) + +def unzip_file(): + with zipfile.ZipFile('/usr/local/airflow/data/microdados_enade_2019.zip', 'r') as zipped: + zipped.extractall('/usr/local/airflow/data') + +unzip_data = PythonOperator( + task_id='unzip_data', + python_callable=unzip_file, + dag=dag +) + +def aplica_filtros(): + cols = ['CO_GRUPO', 'TP_SEXO', 'NU_IDADE', 'NT_GER', 'NT_FG', 'NT_CE', + 'QE_I01','QE_I02','QE_I04','QE_I05','QE_I08'] + enade = pd.read_csv(arquivo, sep=';', decimal=',', usecols=cols) + enade = enade.loc[ + (enade.NU_IDADE > 20) & + (enade.NU_IDADE < 40) & + (enade.NT_GER > 0) + ] + enade.to_csv(data_path + 'enade_filtrado.csv', index=False) + +task_aplica_filtro = PythonOperator( + task_id="aplica_filtro", + python_callable=aplica_filtros, + dag=dag +) + +# Idade centralizada na média +# Idade centralizada ao quadrado + +def constroi_idade_centralizada(): + idade = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['NU_IDADE']) + idade['idadecent'] = idade.NU_IDADE - idade.NU_IDADE.mean() + idade[['idadecent']].to_csv(data_path + "idadecent.csv", index=False) + +def constroi_idade_cent_quad(): + idadecent = pd.read_csv(data_path + "idadecent.csv") + idadecent['idade2'] = idadecent.idadecent ** 2 + idadecent[['idade2']].to_csv(data_path + 'idadequadrado.csv', index=False) + + +task_idade_cent = PythonOperator( + task_id='constroi_idade_centralizada', + python_callable=constroi_idade_centralizada, + dag=dag +) + +task_idade_quad = PythonOperator( + task_id='constroi_idade_ao_quadrado', + python_callable=constroi_idade_cent_quad, + dag=dag +) + +def constroi_est_civil(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I01']) + filtro['estcivil'] = filtro.QE_I01.replace({ + 'A': 'Solteiro', + 'B': 'Casado', + 'C': 'Separado', + 'D': 'Viúvo', + 'E': 'Outro' + }) + filtro[['estcivil']].to_csv(data_path + 'estcivil.csv', index=False) + +task_est_civil = PythonOperator( + task_id='constroi_est_civil', + python_callable=constroi_est_civil, + dag=dag +) + +def constroi_cor(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I02']) + filtro['cor'] = filtro.QE_I02.replace({ + 'A': 'Branca', + 'B': 'Preta', + 'C': 'Amarela', + 'D': 'Parda', + 'E': 'Indígena', + 'F': "", + ' ': "" + }) + filtro[['cor']].to_csv(data_path + 'cor.csv', index=False) + +task_cor = PythonOperator( + task_id='constroi_cor_da_pele', + python_callable=constroi_cor, + dag=dag +) + + +def constroi_escopai(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I04']) + filtro['escopai'] = filtro.QE_I04.replace({ + 'A': 0, + 'B': 1, + 'C': 2, + 'D': 3, + 'E': 4, + 'F': 5 + }) + filtro[['escopai']].to_csv(data_path + 'escopai.csv', index=False) + +task_escopai = PythonOperator( + task_id='constroi_escopai', + python_callable=constroi_escopai, + dag=dag +) + +def constroi_escomae(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I05']) + filtro['escomae'] = filtro.QE_I05.replace({ + 'A': 0, + 'B': 1, + 'C': 2, + 'D': 3, + 'E': 4, + 'F': 5 + }) + filtro[['escomae']].to_csv(data_path + 'escomae.csv', index=False) + +task_escomae = PythonOperator( + task_id='constroi_escomae', + python_callable=constroi_escomae, + dag=dag +) + +def constroi_renda(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I08']) + filtro['renda'] = filtro.QE_I08.replace({ + 'A': 0, + 'B': 1, + 'C': 2, + 'D': 3, + 'E': 4, + 'F': 5, + 'G': 6 + }) + filtro[['renda']].to_csv(data_path + 'renda.csv', index=False) + +task_renda = PythonOperator( + task_id='constroi_renda', + python_callable=constroi_renda, + dag=dag +) + + +# Task de JOIN +def join_data(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv') + idadecent = pd.read_csv(data_path + 'idadecent.csv') + idadeaoquadrado = pd.read_csv(data_path + 'idadequadrado.csv') + estcivil = pd.read_csv(data_path + 'estcivil.csv') + cor = pd.read_csv(data_path + 'cor.csv') + escopai = pd.read_csv(data_path + 'escopai.csv') + escomae = pd.read_csv(data_path + 'escomae.csv') + renda = pd.read_csv(data_path + 'renda.csv') + + final = pd.concat([ + filtro, idadecent, idadeaoquadrado, estcivil, cor, + escopai, escomae, renda + ], + axis=1 + ) + + final.to_csv(data_path + 'enade_tratado.csv', index=False) + print(final) + +task_join = PythonOperator( + task_id='join_data', + python_callable=join_data, + dag=dag +) + +start_preprocessing >> get_data >> unzip_data >> task_aplica_filtro +task_aplica_filtro >> [task_idade_cent, task_est_civil, task_cor, + task_escopai, task_escomae, task_renda] + +task_idade_quad.set_upstream(task_idade_cent) + +task_join.set_upstream([ + task_est_civil, task_cor, task_idade_quad, + task_escopai, task_escomae, task_renda +]) \ No newline at end of file diff --git a/dags/treino05.py b/dags/treino05.py new file mode 100644 index 00000000..4716e533 --- /dev/null +++ b/dags/treino05.py @@ -0,0 +1,262 @@ +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.python_operator import PythonOperator, BranchPythonOperator +from airflow.operators.postgres_operator import PostgresOperator +from datetime import datetime, timedelta +import pandas as pd +import zipfile +import sqlalchemy + +# Constantes +data_path = '/usr/local/airflow/data/microdados_enade_2019/2019/3.DADOS/' +arquivo = data_path + 'microdados_enade_2019.txt' + +default_args = { + 'owner': 'Neylson Crepalde', + "depends_on_past": False, + "start_date": datetime(2020, 12, 2, 23, 15), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=1), +} + +dag = DAG( + "treino-05", + description="DAG completa com entrega no DW", + default_args=default_args, + schedule_interval="*/10 * * * *" +) + +start_preprocessing = BashOperator( + task_id='start_preprocessing', + bash_command='echo "Start Preprocessing! Vai!"', + dag=dag +) + +get_data = BashOperator( + task_id="get-data", + bash_command='curl http://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', + dag=dag +) + +def unzip_file(): + with zipfile.ZipFile('/usr/local/airflow/data/microdados_enade_2019.zip', 'r') as zipped: + zipped.extractall('/usr/local/airflow/data') + +unzip_data = PythonOperator( + task_id='unzip_data', + python_callable=unzip_file, + dag=dag +) + +def aplica_filtros(): + cols = ['CO_GRUPO', 'TP_SEXO', 'NU_IDADE', 'NT_GER', 'NT_FG', 'NT_CE', + 'QE_I01','QE_I02','QE_I04','QE_I05','QE_I08'] + enade = pd.read_csv(arquivo, sep=';', decimal=',', usecols=cols) + enade = enade.loc[ + (enade.NU_IDADE > 20) & + (enade.NU_IDADE < 40) & + (enade.NT_GER > 0) + ] + enade.to_csv(data_path + 'enade_filtrado.csv', index=False) + +task_aplica_filtro = PythonOperator( + task_id="aplica_filtro", + python_callable=aplica_filtros, + dag=dag +) + +# Idade centralizada na média +# Idade centralizada ao quadrado + +def constroi_idade_centralizada(): + idade = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['NU_IDADE']) + idade['idadecent'] = idade.NU_IDADE - idade.NU_IDADE.mean() + idade[['idadecent']].to_csv(data_path + "idadecent.csv", index=False) + +def constroi_idade_cent_quad(): + idadecent = pd.read_csv(data_path + "idadecent.csv") + idadecent['idade2'] = idadecent.idadecent ** 2 + idadecent[['idade2']].to_csv(data_path + 'idadequadrado.csv', index=False) + + +task_idade_cent = PythonOperator( + task_id='constroi_idade_centralizada', + python_callable=constroi_idade_centralizada, + dag=dag +) + +task_idade_quad = PythonOperator( + task_id='constroi_idade_ao_quadrado', + python_callable=constroi_idade_cent_quad, + dag=dag +) + +def constroi_est_civil(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I01']) + filtro['estcivil'] = filtro.QE_I01.replace({ + 'A': 'Solteiro', + 'B': 'Casado', + 'C': 'Separado', + 'D': 'Viúvo', + 'E': 'Outro' + }) + filtro[['estcivil']].to_csv(data_path + 'estcivil.csv', index=False) + +task_est_civil = PythonOperator( + task_id='constroi_est_civil', + python_callable=constroi_est_civil, + dag=dag +) + +def constroi_cor(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I02']) + filtro['cor'] = filtro.QE_I02.replace({ + 'A': 'Branca', + 'B': 'Preta', + 'C': 'Amarela', + 'D': 'Parda', + 'E': 'Indígena', + 'F': "", + ' ': "" + }) + filtro[['cor']].to_csv(data_path + 'cor.csv', index=False) + +task_cor = PythonOperator( + task_id='constroi_cor_da_pele', + python_callable=constroi_cor, + dag=dag +) + + +def constroi_escopai(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I04']) + filtro['escopai'] = filtro.QE_I04.replace({ + 'A': 0, + 'B': 1, + 'C': 2, + 'D': 3, + 'E': 4, + 'F': 5 + }) + filtro[['escopai']].to_csv(data_path + 'escopai.csv', index=False) + +task_escopai = PythonOperator( + task_id='constroi_escopai', + python_callable=constroi_escopai, + dag=dag +) + +def constroi_escomae(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I05']) + filtro['escomae'] = filtro.QE_I05.replace({ + 'A': 0, + 'B': 1, + 'C': 2, + 'D': 3, + 'E': 4, + 'F': 5 + }) + filtro[['escomae']].to_csv(data_path + 'escomae.csv', index=False) + +task_escomae = PythonOperator( + task_id='constroi_escomae', + python_callable=constroi_escomae, + dag=dag +) + +def constroi_renda(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I08']) + filtro['renda'] = filtro.QE_I08.replace({ + 'A': 0, + 'B': 1, + 'C': 2, + 'D': 3, + 'E': 4, + 'F': 5, + 'G': 6 + }) + filtro[['renda']].to_csv(data_path + 'renda.csv', index=False) + +task_renda = PythonOperator( + task_id='constroi_renda', + python_callable=constroi_renda, + dag=dag +) + + +# Task de JOIN +def join_data(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv') + idadecent = pd.read_csv(data_path + 'idadecent.csv') + idadeaoquadrado = pd.read_csv(data_path + 'idadequadrado.csv') + estcivil = pd.read_csv(data_path + 'estcivil.csv') + cor = pd.read_csv(data_path + 'cor.csv') + escopai = pd.read_csv(data_path + 'escopai.csv') + escomae = pd.read_csv(data_path + 'escomae.csv') + renda = pd.read_csv(data_path + 'renda.csv') + + final = pd.concat([ + filtro, idadecent, idadeaoquadrado, estcivil, cor, + escopai, escomae, renda + ], + axis=1 + ) + + final.to_csv(data_path + 'enade_tratado.csv', index=False) + +task_join = PythonOperator( + task_id='join_data', + python_callable=join_data, + dag=dag +) + +task_create_schema = PostgresOperator( + task_id='cria_schema', + sql=""" + CREATE TABLE tratado( + CO_GRUPO integer, + NU_IDADE integer, + TP_SEXO varchar(10), + NT_GER numeric, + NT_FG numeric, + NT_CE numeric, + QE_I01 varchar(2), + QE_I02 varchar(2), + QE_I04 varchar(2), + QE_I05 varchar(2), + QE_I08 varchar(2), + idadecent numeric, + idade2 numeric, + estcivil varchar(20), + cor varchar(10), + escopai integer, + escomae integer, + renda integer + ) + """, + dag=dag +) + +task_escreve_dw = PostgresOperator( + task_id='escreve_dw', + #python_callable=escreve_dw, + sql=f"COPY tratado FROM '{data_path}enade_tratado.csv' WITH (FORMAT CSV);", + dag=dag +) + + +start_preprocessing >> get_data >> unzip_data >> task_aplica_filtro +task_aplica_filtro >> [task_idade_cent, task_est_civil, task_cor, + task_escopai, task_escomae, task_renda] + +task_idade_quad.set_upstream(task_idade_cent) + +task_join.set_upstream([ + task_est_civil, task_cor, task_idade_quad, + task_escopai, task_escomae, task_renda +]) + +task_join >> task_create_schema >> task_escreve_dw \ No newline at end of file diff --git a/docker-compose-CeleryExecutor.yml b/docker-compose.yml similarity index 91% rename from docker-compose-CeleryExecutor.yml rename to docker-compose.yml index de4f5dac..e3c61549 100644 --- a/docker-compose-CeleryExecutor.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ -version: '2.1' +version: '2.2' services: redis: image: 'redis:5.0.5' @@ -16,7 +16,7 @@ services: # - ./pgdata:/var/lib/postgresql/data/pgdata webserver: - image: puckel/docker-airflow:1.10.9 + image: ney/airflow:2.0.0 restart: always depends_on: - postgres @@ -43,7 +43,7 @@ services: retries: 3 flower: - image: puckel/docker-airflow:1.10.9 + image: ney/airflow:2.0.0 restart: always depends_on: - redis @@ -52,10 +52,10 @@ services: # - REDIS_PASSWORD=redispass ports: - "5555:5555" - command: flower + command: celery flower scheduler: - image: puckel/docker-airflow:1.10.9 + image: ney/airflow:2.0.0 restart: always depends_on: - webserver @@ -74,7 +74,7 @@ services: command: scheduler worker: - image: puckel/docker-airflow:1.10.9 + image: ney/airflow:2.0.0 restart: always depends_on: - scheduler @@ -89,4 +89,4 @@ services: # - POSTGRES_PASSWORD=airflow # - POSTGRES_DB=airflow # - REDIS_PASSWORD=redispass - command: worker + command: celery worker diff --git a/script/entrypoint.sh b/script/entrypoint.sh index 166f4837..524d0f5f 100755 --- a/script/entrypoint.sh +++ b/script/entrypoint.sh @@ -109,19 +109,20 @@ fi case "$1" in webserver) - airflow initdb + airflow db init + airflow users create --username airflow --password airflow --firstname Peter --lastname Parker --role Admin --email spiderman@superhero.org if [ "$AIRFLOW__CORE__EXECUTOR" = "LocalExecutor" ] || [ "$AIRFLOW__CORE__EXECUTOR" = "SequentialExecutor" ]; then # With the "Local" and "Sequential" executors it should all run in one container. airflow scheduler & fi exec airflow webserver ;; - worker|scheduler) - # Give the webserver time to run initdb. + "celery worker"|scheduler) + # Give the webserver time to run db init. sleep 10 exec airflow "$@" ;; - flower) + "celery flower") sleep 10 exec airflow "$@" ;; From be381af5b949ed421665820346bedf4958edccb8 Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Sat, 19 Dec 2020 12:56:59 -0300 Subject: [PATCH 02/20] fix worker connection to redis --- dags/treino01.py | 41 -------- dags/treino02.py | 53 ---------- dags/treino03.py | 80 --------------- dags/treino04.py | 224 ---------------------------------------- dags/treino05.py | 262 ----------------------------------------------- 5 files changed, 660 deletions(-) delete mode 100644 dags/treino01.py delete mode 100644 dags/treino02.py delete mode 100644 dags/treino03.py delete mode 100644 dags/treino04.py delete mode 100644 dags/treino05.py diff --git a/dags/treino01.py b/dags/treino01.py deleted file mode 100644 index 98c1e974..00000000 --- a/dags/treino01.py +++ /dev/null @@ -1,41 +0,0 @@ -from airflow import DAG -from airflow.operators.bash_operator import BashOperator -from airflow.operators.python_operator import PythonOperator -from datetime import datetime, timedelta - -default_args = { - 'owner': 'Neylson Crepalde', - "depends_on_past": False, - "start_date": datetime(2020, 11, 14), - "email": ["airflow@airflow.com"], - "email_on_failure": False, - "email_on_retry": False, - "retries": 1, - "retry_delay": timedelta(minutes=2), -} - -dag = DAG( - "treino-01", - description="Básico de Bash operators e Python operators", - default_args=default_args, - schedule_interval=timedelta(minutes=1) -) - -def hello_word(): - print("Hello Airflow from Python") - - -hello_bash = BashOperator( - task_id="hello-bash", - bash_command='echo "Hello Airflow from bash"', - dag=dag -) - -hello_python = PythonOperator( - task_id='hello-python', - python_callable=hello_word, - dag=dag -) - - -hello_bash >> hello_python diff --git a/dags/treino02.py b/dags/treino02.py deleted file mode 100644 index a8903b8d..00000000 --- a/dags/treino02.py +++ /dev/null @@ -1,53 +0,0 @@ -from airflow import DAG -from airflow.operators.bash_operator import BashOperator -from airflow.operators.python_operator import PythonOperator -from datetime import datetime, timedelta -import pandas as pd - -default_args = { - 'owner': 'Neylson Crepalde', - "depends_on_past": False, - "start_date": datetime(2020, 11, 14, 23, 50), - "email": ["airflow@airflow.com"], - "email_on_failure": False, - "email_on_retry": False, - "retries": 1, - "retry_delay": timedelta(minutes=1), -} - -dag = DAG( - "treino-02", - description="Get Titanic data from internet and calculate mean age", - default_args=default_args, - schedule_interval=timedelta(minutes=5) -) - -get_data = BashOperator( - task_id="get-data", - bash_command='curl https://raw.githubusercontent.com/A3Data/hermione/master/hermione/file_text/train.csv -o ~/train.csv', - dag=dag -) - -def calculate_mean_age(): - df = pd.read_csv('~/train.csv') - med = df.Age.mean() - return med - -task_calculate_mean = PythonOperator( - task_id='calculate-mean-age', - python_callable=calculate_mean_age, - dag=dag -) - -def print_age(**context): - value = context['task_instance'].xcom_pull(task_ids='calculate-mean-age') - print(f"Mean age is {value}") - -task_mean_age = PythonOperator( - task_id="say-mean-age", - python_callable=print_age, - provide_context=True, - dag=dag -) - -get_data >> task_calculate_mean >> task_mean_age \ No newline at end of file diff --git a/dags/treino03.py b/dags/treino03.py deleted file mode 100644 index 5fa19f9a..00000000 --- a/dags/treino03.py +++ /dev/null @@ -1,80 +0,0 @@ -from airflow import DAG -from airflow.operators.bash_operator import BashOperator -from airflow.operators.python_operator import PythonOperator, BranchPythonOperator -from datetime import datetime, timedelta -import pandas as pd -import random - -default_args = { - 'owner': 'Neylson Crepalde', - "depends_on_past": False, - "start_date": datetime(2020, 11, 15, 16, 20), - "email": ["airflow@airflow.com"], - "email_on_failure": False, - "email_on_retry": False, - "retries": 1, - "retry_delay": timedelta(minutes=1), -} - -dag = DAG( - "treino-03", - description="Pega dados do Titanic e calcula idade média para homens ou mulheres", - default_args=default_args, - schedule_interval=timedelta(minutes=2) -) - -get_data = BashOperator( - task_id="get-data", - bash_command='curl https://raw.githubusercontent.com/A3Data/hermione/master/hermione/file_text/train.csv -o /usr/local/airflow/data/train.csv', - dag=dag -) - - -def sorteia_h_m(): - return random.choice(['male', 'female']) - -escolhe_h_m = PythonOperator( - task_id='escolhe-h-m', - python_callable=sorteia_h_m, - dag=dag -) - -def MouF(**context): - value = context['task_instance'].xcom_pull(task_ids='escolhe-h-m') - if value == 'male': - return 'branch_homem' - if value == 'female': - return 'branch_mulher' - -male_female = BranchPythonOperator( - task_id='condicional', - python_callable=MouF, - provide_context=True, - dag=dag -) - - -def mean_homen(): - df = pd.read_csv('/usr/local/airflow/data/train.csv') - df = df.loc[df.Sex == 'male'] - print(f"Média de idade dos homens no Titanic: {df.Age.mean()}") - -branch_homem = PythonOperator( - task_id='branch_homem', - python_callable=mean_homen, - dag=dag -) - -def mean_mulher(): - df = pd.read_csv('/usr/local/airflow/data/train.csv') - df = df.loc[df.Sex == 'female'] - print(f"Média de idade das mulheres no Titanic: {df.Age.mean()}") - -branch_mulher = PythonOperator( - task_id='branch_mulher', - python_callable=mean_mulher, - dag=dag -) - - -get_data >> escolhe_h_m >> male_female >> [branch_homem, branch_mulher] diff --git a/dags/treino04.py b/dags/treino04.py deleted file mode 100644 index f2dd834c..00000000 --- a/dags/treino04.py +++ /dev/null @@ -1,224 +0,0 @@ -from airflow import DAG -from airflow.operators.bash_operator import BashOperator -from airflow.operators.python_operator import PythonOperator, BranchPythonOperator -from datetime import datetime, timedelta -import pandas as pd -import zipfile - -# Constantes -data_path = '/usr/local/airflow/data/microdados_enade_2019/2019/3.DADOS/' -arquivo = data_path + 'microdados_enade_2019.txt' - -default_args = { - 'owner': 'Neylson Crepalde', - "depends_on_past": False, - "start_date": datetime(2020, 11, 15, 17), - "email": ["airflow@airflow.com"], - "email_on_failure": False, - "email_on_retry": False, - "retries": 1, - "retry_delay": timedelta(minutes=1), -} - -dag = DAG( - "treino-04", - description="Paralelismos", - default_args=default_args, - schedule_interval="*/10 * * * *" -) - -start_preprocessing = BashOperator( - task_id='start_preprocessing', - bash_command='echo "Start Preprocessing! Vai!"', - dag=dag -) - -get_data = BashOperator( - task_id="get-data", - bash_command='curl http://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', - dag=dag -) - -def unzip_file(): - with zipfile.ZipFile('/usr/local/airflow/data/microdados_enade_2019.zip', 'r') as zipped: - zipped.extractall('/usr/local/airflow/data') - -unzip_data = PythonOperator( - task_id='unzip_data', - python_callable=unzip_file, - dag=dag -) - -def aplica_filtros(): - cols = ['CO_GRUPO', 'TP_SEXO', 'NU_IDADE', 'NT_GER', 'NT_FG', 'NT_CE', - 'QE_I01','QE_I02','QE_I04','QE_I05','QE_I08'] - enade = pd.read_csv(arquivo, sep=';', decimal=',', usecols=cols) - enade = enade.loc[ - (enade.NU_IDADE > 20) & - (enade.NU_IDADE < 40) & - (enade.NT_GER > 0) - ] - enade.to_csv(data_path + 'enade_filtrado.csv', index=False) - -task_aplica_filtro = PythonOperator( - task_id="aplica_filtro", - python_callable=aplica_filtros, - dag=dag -) - -# Idade centralizada na média -# Idade centralizada ao quadrado - -def constroi_idade_centralizada(): - idade = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['NU_IDADE']) - idade['idadecent'] = idade.NU_IDADE - idade.NU_IDADE.mean() - idade[['idadecent']].to_csv(data_path + "idadecent.csv", index=False) - -def constroi_idade_cent_quad(): - idadecent = pd.read_csv(data_path + "idadecent.csv") - idadecent['idade2'] = idadecent.idadecent ** 2 - idadecent[['idade2']].to_csv(data_path + 'idadequadrado.csv', index=False) - - -task_idade_cent = PythonOperator( - task_id='constroi_idade_centralizada', - python_callable=constroi_idade_centralizada, - dag=dag -) - -task_idade_quad = PythonOperator( - task_id='constroi_idade_ao_quadrado', - python_callable=constroi_idade_cent_quad, - dag=dag -) - -def constroi_est_civil(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I01']) - filtro['estcivil'] = filtro.QE_I01.replace({ - 'A': 'Solteiro', - 'B': 'Casado', - 'C': 'Separado', - 'D': 'Viúvo', - 'E': 'Outro' - }) - filtro[['estcivil']].to_csv(data_path + 'estcivil.csv', index=False) - -task_est_civil = PythonOperator( - task_id='constroi_est_civil', - python_callable=constroi_est_civil, - dag=dag -) - -def constroi_cor(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I02']) - filtro['cor'] = filtro.QE_I02.replace({ - 'A': 'Branca', - 'B': 'Preta', - 'C': 'Amarela', - 'D': 'Parda', - 'E': 'Indígena', - 'F': "", - ' ': "" - }) - filtro[['cor']].to_csv(data_path + 'cor.csv', index=False) - -task_cor = PythonOperator( - task_id='constroi_cor_da_pele', - python_callable=constroi_cor, - dag=dag -) - - -def constroi_escopai(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I04']) - filtro['escopai'] = filtro.QE_I04.replace({ - 'A': 0, - 'B': 1, - 'C': 2, - 'D': 3, - 'E': 4, - 'F': 5 - }) - filtro[['escopai']].to_csv(data_path + 'escopai.csv', index=False) - -task_escopai = PythonOperator( - task_id='constroi_escopai', - python_callable=constroi_escopai, - dag=dag -) - -def constroi_escomae(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I05']) - filtro['escomae'] = filtro.QE_I05.replace({ - 'A': 0, - 'B': 1, - 'C': 2, - 'D': 3, - 'E': 4, - 'F': 5 - }) - filtro[['escomae']].to_csv(data_path + 'escomae.csv', index=False) - -task_escomae = PythonOperator( - task_id='constroi_escomae', - python_callable=constroi_escomae, - dag=dag -) - -def constroi_renda(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I08']) - filtro['renda'] = filtro.QE_I08.replace({ - 'A': 0, - 'B': 1, - 'C': 2, - 'D': 3, - 'E': 4, - 'F': 5, - 'G': 6 - }) - filtro[['renda']].to_csv(data_path + 'renda.csv', index=False) - -task_renda = PythonOperator( - task_id='constroi_renda', - python_callable=constroi_renda, - dag=dag -) - - -# Task de JOIN -def join_data(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv') - idadecent = pd.read_csv(data_path + 'idadecent.csv') - idadeaoquadrado = pd.read_csv(data_path + 'idadequadrado.csv') - estcivil = pd.read_csv(data_path + 'estcivil.csv') - cor = pd.read_csv(data_path + 'cor.csv') - escopai = pd.read_csv(data_path + 'escopai.csv') - escomae = pd.read_csv(data_path + 'escomae.csv') - renda = pd.read_csv(data_path + 'renda.csv') - - final = pd.concat([ - filtro, idadecent, idadeaoquadrado, estcivil, cor, - escopai, escomae, renda - ], - axis=1 - ) - - final.to_csv(data_path + 'enade_tratado.csv', index=False) - print(final) - -task_join = PythonOperator( - task_id='join_data', - python_callable=join_data, - dag=dag -) - -start_preprocessing >> get_data >> unzip_data >> task_aplica_filtro -task_aplica_filtro >> [task_idade_cent, task_est_civil, task_cor, - task_escopai, task_escomae, task_renda] - -task_idade_quad.set_upstream(task_idade_cent) - -task_join.set_upstream([ - task_est_civil, task_cor, task_idade_quad, - task_escopai, task_escomae, task_renda -]) \ No newline at end of file diff --git a/dags/treino05.py b/dags/treino05.py deleted file mode 100644 index 4716e533..00000000 --- a/dags/treino05.py +++ /dev/null @@ -1,262 +0,0 @@ -from airflow import DAG -from airflow.operators.bash_operator import BashOperator -from airflow.operators.python_operator import PythonOperator, BranchPythonOperator -from airflow.operators.postgres_operator import PostgresOperator -from datetime import datetime, timedelta -import pandas as pd -import zipfile -import sqlalchemy - -# Constantes -data_path = '/usr/local/airflow/data/microdados_enade_2019/2019/3.DADOS/' -arquivo = data_path + 'microdados_enade_2019.txt' - -default_args = { - 'owner': 'Neylson Crepalde', - "depends_on_past": False, - "start_date": datetime(2020, 12, 2, 23, 15), - "email": ["airflow@airflow.com"], - "email_on_failure": False, - "email_on_retry": False, - "retries": 1, - "retry_delay": timedelta(minutes=1), -} - -dag = DAG( - "treino-05", - description="DAG completa com entrega no DW", - default_args=default_args, - schedule_interval="*/10 * * * *" -) - -start_preprocessing = BashOperator( - task_id='start_preprocessing', - bash_command='echo "Start Preprocessing! Vai!"', - dag=dag -) - -get_data = BashOperator( - task_id="get-data", - bash_command='curl http://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', - dag=dag -) - -def unzip_file(): - with zipfile.ZipFile('/usr/local/airflow/data/microdados_enade_2019.zip', 'r') as zipped: - zipped.extractall('/usr/local/airflow/data') - -unzip_data = PythonOperator( - task_id='unzip_data', - python_callable=unzip_file, - dag=dag -) - -def aplica_filtros(): - cols = ['CO_GRUPO', 'TP_SEXO', 'NU_IDADE', 'NT_GER', 'NT_FG', 'NT_CE', - 'QE_I01','QE_I02','QE_I04','QE_I05','QE_I08'] - enade = pd.read_csv(arquivo, sep=';', decimal=',', usecols=cols) - enade = enade.loc[ - (enade.NU_IDADE > 20) & - (enade.NU_IDADE < 40) & - (enade.NT_GER > 0) - ] - enade.to_csv(data_path + 'enade_filtrado.csv', index=False) - -task_aplica_filtro = PythonOperator( - task_id="aplica_filtro", - python_callable=aplica_filtros, - dag=dag -) - -# Idade centralizada na média -# Idade centralizada ao quadrado - -def constroi_idade_centralizada(): - idade = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['NU_IDADE']) - idade['idadecent'] = idade.NU_IDADE - idade.NU_IDADE.mean() - idade[['idadecent']].to_csv(data_path + "idadecent.csv", index=False) - -def constroi_idade_cent_quad(): - idadecent = pd.read_csv(data_path + "idadecent.csv") - idadecent['idade2'] = idadecent.idadecent ** 2 - idadecent[['idade2']].to_csv(data_path + 'idadequadrado.csv', index=False) - - -task_idade_cent = PythonOperator( - task_id='constroi_idade_centralizada', - python_callable=constroi_idade_centralizada, - dag=dag -) - -task_idade_quad = PythonOperator( - task_id='constroi_idade_ao_quadrado', - python_callable=constroi_idade_cent_quad, - dag=dag -) - -def constroi_est_civil(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I01']) - filtro['estcivil'] = filtro.QE_I01.replace({ - 'A': 'Solteiro', - 'B': 'Casado', - 'C': 'Separado', - 'D': 'Viúvo', - 'E': 'Outro' - }) - filtro[['estcivil']].to_csv(data_path + 'estcivil.csv', index=False) - -task_est_civil = PythonOperator( - task_id='constroi_est_civil', - python_callable=constroi_est_civil, - dag=dag -) - -def constroi_cor(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I02']) - filtro['cor'] = filtro.QE_I02.replace({ - 'A': 'Branca', - 'B': 'Preta', - 'C': 'Amarela', - 'D': 'Parda', - 'E': 'Indígena', - 'F': "", - ' ': "" - }) - filtro[['cor']].to_csv(data_path + 'cor.csv', index=False) - -task_cor = PythonOperator( - task_id='constroi_cor_da_pele', - python_callable=constroi_cor, - dag=dag -) - - -def constroi_escopai(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I04']) - filtro['escopai'] = filtro.QE_I04.replace({ - 'A': 0, - 'B': 1, - 'C': 2, - 'D': 3, - 'E': 4, - 'F': 5 - }) - filtro[['escopai']].to_csv(data_path + 'escopai.csv', index=False) - -task_escopai = PythonOperator( - task_id='constroi_escopai', - python_callable=constroi_escopai, - dag=dag -) - -def constroi_escomae(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I05']) - filtro['escomae'] = filtro.QE_I05.replace({ - 'A': 0, - 'B': 1, - 'C': 2, - 'D': 3, - 'E': 4, - 'F': 5 - }) - filtro[['escomae']].to_csv(data_path + 'escomae.csv', index=False) - -task_escomae = PythonOperator( - task_id='constroi_escomae', - python_callable=constroi_escomae, - dag=dag -) - -def constroi_renda(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I08']) - filtro['renda'] = filtro.QE_I08.replace({ - 'A': 0, - 'B': 1, - 'C': 2, - 'D': 3, - 'E': 4, - 'F': 5, - 'G': 6 - }) - filtro[['renda']].to_csv(data_path + 'renda.csv', index=False) - -task_renda = PythonOperator( - task_id='constroi_renda', - python_callable=constroi_renda, - dag=dag -) - - -# Task de JOIN -def join_data(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv') - idadecent = pd.read_csv(data_path + 'idadecent.csv') - idadeaoquadrado = pd.read_csv(data_path + 'idadequadrado.csv') - estcivil = pd.read_csv(data_path + 'estcivil.csv') - cor = pd.read_csv(data_path + 'cor.csv') - escopai = pd.read_csv(data_path + 'escopai.csv') - escomae = pd.read_csv(data_path + 'escomae.csv') - renda = pd.read_csv(data_path + 'renda.csv') - - final = pd.concat([ - filtro, idadecent, idadeaoquadrado, estcivil, cor, - escopai, escomae, renda - ], - axis=1 - ) - - final.to_csv(data_path + 'enade_tratado.csv', index=False) - -task_join = PythonOperator( - task_id='join_data', - python_callable=join_data, - dag=dag -) - -task_create_schema = PostgresOperator( - task_id='cria_schema', - sql=""" - CREATE TABLE tratado( - CO_GRUPO integer, - NU_IDADE integer, - TP_SEXO varchar(10), - NT_GER numeric, - NT_FG numeric, - NT_CE numeric, - QE_I01 varchar(2), - QE_I02 varchar(2), - QE_I04 varchar(2), - QE_I05 varchar(2), - QE_I08 varchar(2), - idadecent numeric, - idade2 numeric, - estcivil varchar(20), - cor varchar(10), - escopai integer, - escomae integer, - renda integer - ) - """, - dag=dag -) - -task_escreve_dw = PostgresOperator( - task_id='escreve_dw', - #python_callable=escreve_dw, - sql=f"COPY tratado FROM '{data_path}enade_tratado.csv' WITH (FORMAT CSV);", - dag=dag -) - - -start_preprocessing >> get_data >> unzip_data >> task_aplica_filtro -task_aplica_filtro >> [task_idade_cent, task_est_civil, task_cor, - task_escopai, task_escomae, task_renda] - -task_idade_quad.set_upstream(task_idade_cent) - -task_join.set_upstream([ - task_est_civil, task_cor, task_idade_quad, - task_escopai, task_escomae, task_renda -]) - -task_join >> task_create_schema >> task_escreve_dw \ No newline at end of file From 06ea87822a222c2a7210d9b6480a4055407f3807 Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Sat, 19 Dec 2020 12:58:06 -0300 Subject: [PATCH 03/20] rename docker-compose-CeleryExecutor.yml --- docker-compose.yml => docker-compose-CeleryExecutor.yml | 3 +++ script/entrypoint.sh | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) rename docker-compose.yml => docker-compose-CeleryExecutor.yml (95%) diff --git a/docker-compose.yml b/docker-compose-CeleryExecutor.yml similarity index 95% rename from docker-compose.yml rename to docker-compose-CeleryExecutor.yml index e3c61549..140293f9 100644 --- a/docker-compose.yml +++ b/docker-compose-CeleryExecutor.yml @@ -31,6 +31,7 @@ services: # - REDIS_PASSWORD=redispass volumes: - ./dags:/usr/local/airflow/dags + - ./data:/usr/local/airflow/data # Uncomment to include custom plugins # - ./plugins:/usr/local/airflow/plugins ports: @@ -61,6 +62,7 @@ services: - webserver volumes: - ./dags:/usr/local/airflow/dags + - ./data:/usr/local/airflow/data # Uncomment to include custom plugins # - ./plugins:/usr/local/airflow/plugins environment: @@ -80,6 +82,7 @@ services: - scheduler volumes: - ./dags:/usr/local/airflow/dags + - ./data:/usr/local/airflow/data # Uncomment to include custom plugins # - ./plugins:/usr/local/airflow/plugins environment: diff --git a/script/entrypoint.sh b/script/entrypoint.sh index 524d0f5f..862c9d55 100755 --- a/script/entrypoint.sh +++ b/script/entrypoint.sh @@ -117,12 +117,12 @@ case "$1" in fi exec airflow webserver ;; - "celery worker"|scheduler) + celery|scheduler) # Give the webserver time to run db init. sleep 10 exec airflow "$@" ;; - "celery flower") + celery) sleep 10 exec airflow "$@" ;; From 70a06d026322cea1f035e9b8bbfe2dd6997caa66 Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Sat, 19 Dec 2020 13:08:32 -0300 Subject: [PATCH 04/20] fix image names --- docker-compose-CeleryExecutor.yml | 8 ++++---- docker-compose-LocalExecutor.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose-CeleryExecutor.yml b/docker-compose-CeleryExecutor.yml index 140293f9..9978f8b4 100644 --- a/docker-compose-CeleryExecutor.yml +++ b/docker-compose-CeleryExecutor.yml @@ -16,7 +16,7 @@ services: # - ./pgdata:/var/lib/postgresql/data/pgdata webserver: - image: ney/airflow:2.0.0 + image: puckel/docker-airflow:2.0.0 restart: always depends_on: - postgres @@ -44,7 +44,7 @@ services: retries: 3 flower: - image: ney/airflow:2.0.0 + image: puckel/docker-airflow:2.0.0 restart: always depends_on: - redis @@ -56,7 +56,7 @@ services: command: celery flower scheduler: - image: ney/airflow:2.0.0 + image: puckel/docker-airflow:2.0.0 restart: always depends_on: - webserver @@ -76,7 +76,7 @@ services: command: scheduler worker: - image: ney/airflow:2.0.0 + image: puckel/docker-airflow:2.0.0 restart: always depends_on: - scheduler diff --git a/docker-compose-LocalExecutor.yml b/docker-compose-LocalExecutor.yml index 26e9e92e..33311c00 100644 --- a/docker-compose-LocalExecutor.yml +++ b/docker-compose-LocalExecutor.yml @@ -1,4 +1,4 @@ -version: '3.7' +version: '3.8' services: postgres: image: postgres:9.6 @@ -12,7 +12,7 @@ services: max-file: "3" webserver: - image: puckel/docker-airflow:1.10.9 + image: puckel/docker-airflow:2.0.0 restart: always depends_on: - postgres From b7a3897c63fb0644a2fb3aaa8ad5d51245d4eceb Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Sat, 19 Dec 2020 13:11:35 -0300 Subject: [PATCH 05/20] update to airflow 2.0.0 and Python 3.8 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 922e51a7..b058e6cc 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ This repository contains **Dockerfile** of [apache-airflow](https://github.com/a ## Informations -* Based on Python (3.7-slim-buster) official Image [python:3.7-slim-buster](https://hub.docker.com/_/python/) and uses the official [Postgres](https://hub.docker.com/_/postgres/) as backend and [Redis](https://hub.docker.com/_/redis/) as queue +* Based on Python (3.8-slim-buster) official Image [python:3.8-slim-buster](https://hub.docker.com/_/python/) and uses the official [Postgres](https://hub.docker.com/_/postgres/) as backend and [Redis](https://hub.docker.com/_/redis/) as queue * Install [Docker](https://www.docker.com/) * Install [Docker Compose](https://docs.docker.com/compose/install/) * Following the Airflow release from [Python Package Index](https://pypi.python.org/pypi/apache-airflow) From 22c29412253b24f89f4c54193c0a5d139f6ac5ba Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Sat, 19 Dec 2020 13:13:58 -0300 Subject: [PATCH 06/20] default webserver credentials --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index b058e6cc..f6d6cd23 100644 --- a/README.md +++ b/README.md @@ -98,6 +98,9 @@ In order to incorporate plugins into your docker container - Airflow: [localhost:8080](http://localhost:8080/) - Flower: [localhost:5555](http://localhost:5555/) +To log into airflow webserver, the default credentials are +- username: airflow +- password: airflow ## Scale the number of workers From 64730882ee148108f56116560739c0d5055b8a79 Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Sat, 19 Dec 2020 13:19:59 -0300 Subject: [PATCH 07/20] leave the same volumes as puckels --- docker-compose-CeleryExecutor.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker-compose-CeleryExecutor.yml b/docker-compose-CeleryExecutor.yml index 9978f8b4..af6ba37d 100644 --- a/docker-compose-CeleryExecutor.yml +++ b/docker-compose-CeleryExecutor.yml @@ -31,7 +31,6 @@ services: # - REDIS_PASSWORD=redispass volumes: - ./dags:/usr/local/airflow/dags - - ./data:/usr/local/airflow/data # Uncomment to include custom plugins # - ./plugins:/usr/local/airflow/plugins ports: @@ -62,7 +61,6 @@ services: - webserver volumes: - ./dags:/usr/local/airflow/dags - - ./data:/usr/local/airflow/data # Uncomment to include custom plugins # - ./plugins:/usr/local/airflow/plugins environment: @@ -82,7 +80,6 @@ services: - scheduler volumes: - ./dags:/usr/local/airflow/dags - - ./data:/usr/local/airflow/data # Uncomment to include custom plugins # - ./plugins:/usr/local/airflow/plugins environment: From f4bb87d5192db382bd68b4799f9686eaea022b91 Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Sat, 19 Dec 2020 21:46:46 -0300 Subject: [PATCH 08/20] my dags and neylson image --- dags/treino01.py | 41 ++++++ dags/treino02.py | 53 +++++++ dags/treino03.py | 80 +++++++++++ dags/treino04.py | 225 ++++++++++++++++++++++++++++++ docker-compose-CeleryExecutor.yml | 8 +- docker-compose-LocalExecutor.yml | 2 +- 6 files changed, 404 insertions(+), 5 deletions(-) create mode 100644 dags/treino01.py create mode 100644 dags/treino02.py create mode 100644 dags/treino03.py create mode 100644 dags/treino04.py diff --git a/dags/treino01.py b/dags/treino01.py new file mode 100644 index 00000000..550e457e --- /dev/null +++ b/dags/treino01.py @@ -0,0 +1,41 @@ +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.python_operator import PythonOperator +from datetime import datetime, timedelta + +default_args = { + 'owner': 'Neylson Crepalde', + "depends_on_past": False, + "start_date": datetime(2020, 11, 14), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=2), +} + +dag = DAG( + "treino-01", + description="Básico de Bash operators e Python operators", + default_args=default_args, + schedule_interval=timedelta(minutes=1) +) + +def hello_word(): + print("Hello Airflow from Python") + + +hello_bash = BashOperator( + task_id="hello-bash", + bash_command='echo "Hello Airflow from bash"', + dag=dag +) + +hello_python = PythonOperator( + task_id='hello-python', + python_callable=hello_word, + dag=dag +) + + +hello_bash >> hello_python \ No newline at end of file diff --git a/dags/treino02.py b/dags/treino02.py new file mode 100644 index 00000000..a8903b8d --- /dev/null +++ b/dags/treino02.py @@ -0,0 +1,53 @@ +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.python_operator import PythonOperator +from datetime import datetime, timedelta +import pandas as pd + +default_args = { + 'owner': 'Neylson Crepalde', + "depends_on_past": False, + "start_date": datetime(2020, 11, 14, 23, 50), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=1), +} + +dag = DAG( + "treino-02", + description="Get Titanic data from internet and calculate mean age", + default_args=default_args, + schedule_interval=timedelta(minutes=5) +) + +get_data = BashOperator( + task_id="get-data", + bash_command='curl https://raw.githubusercontent.com/A3Data/hermione/master/hermione/file_text/train.csv -o ~/train.csv', + dag=dag +) + +def calculate_mean_age(): + df = pd.read_csv('~/train.csv') + med = df.Age.mean() + return med + +task_calculate_mean = PythonOperator( + task_id='calculate-mean-age', + python_callable=calculate_mean_age, + dag=dag +) + +def print_age(**context): + value = context['task_instance'].xcom_pull(task_ids='calculate-mean-age') + print(f"Mean age is {value}") + +task_mean_age = PythonOperator( + task_id="say-mean-age", + python_callable=print_age, + provide_context=True, + dag=dag +) + +get_data >> task_calculate_mean >> task_mean_age \ No newline at end of file diff --git a/dags/treino03.py b/dags/treino03.py new file mode 100644 index 00000000..e2e2f23d --- /dev/null +++ b/dags/treino03.py @@ -0,0 +1,80 @@ +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.python_operator import PythonOperator, BranchPythonOperator +from datetime import datetime, timedelta +import pandas as pd +import random + +default_args = { + 'owner': 'Neylson Crepalde', + "depends_on_past": False, + "start_date": datetime(2020, 11, 15, 16, 20), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=1), +} + +dag = DAG( + "treino-03", + description="Pega dados do Titanic e calcula idade média para homens ou mulheres", + default_args=default_args, + schedule_interval=timedelta(minutes=2) +) + +get_data = BashOperator( + task_id="get-data", + bash_command='curl https://raw.githubusercontent.com/A3Data/hermione/master/hermione/file_text/train.csv -o /usr/local/airflow/train.csv', + dag=dag +) + + +def sorteia_h_m(): + return random.choice(['male', 'female']) + +escolhe_h_m = PythonOperator( + task_id='escolhe-h-m', + python_callable=sorteia_h_m, + dag=dag +) + +def MouF(**context): + value = context['task_instance'].xcom_pull(task_ids='escolhe-h-m') + if value == 'male': + return 'branch_homem' + if value == 'female': + return 'branch_mulher' + +male_female = BranchPythonOperator( + task_id='condicional', + python_callable=MouF, + provide_context=True, + dag=dag +) + + +def mean_homen(): + df = pd.read_csv('/usr/local/airflow/train.csv') + df = df.loc[df.Sex == 'male'] + print(f"Média de idade dos homens no Titanic: {df.Age.mean()}") + +branch_homem = PythonOperator( + task_id='branch_homem', + python_callable=mean_homen, + dag=dag +) + +def mean_mulher(): + df = pd.read_csv('/usr/local/airflow/train.csv') + df = df.loc[df.Sex == 'female'] + print(f"Média de idade das mulheres no Titanic: {df.Age.mean()}") + +branch_mulher = PythonOperator( + task_id='branch_mulher', + python_callable=mean_mulher, + dag=dag +) + + +get_data >> escolhe_h_m >> male_female >> [branch_homem, branch_mulher] \ No newline at end of file diff --git a/dags/treino04.py b/dags/treino04.py new file mode 100644 index 00000000..5690e418 --- /dev/null +++ b/dags/treino04.py @@ -0,0 +1,225 @@ +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.python_operator import PythonOperator, BranchPythonOperator +from datetime import datetime, timedelta +import os +import pandas as pd +import zipfile + +# Constantes +data_path = '/usr/local/airflow/data/microdados_enade_2019/2019/3.DADOS/' +arquivo = data_path + 'microdados_enade_2019.txt' + +default_args = { + 'owner': 'Neylson Crepalde', + "depends_on_past": False, + "start_date": datetime(2020, 11, 15, 17), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=1), +} + +dag = DAG( + "treino-04", + description="Paralelismos", + default_args=default_args, + schedule_interval="*/10 * * * *" +) + +start_preprocessing = BashOperator( + task_id='start_preprocessing', + bash_command='echo "Start Preprocessing! Vai!"', + dag=dag +) + +get_data = BashOperator( + task_id="get-data", + bash_command='mkdir -p /usr/local/airflow/data && curl http://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', + dag=dag +) + +def unzip_file(): + with zipfile.ZipFile('/usr/local/airflow/data/microdados_enade_2019.zip', 'r') as zipped: + zipped.extractall('/usr/local/airflow/data') + +unzip_data = PythonOperator( + task_id='unzip_data', + python_callable=unzip_file, + dag=dag +) + +def aplica_filtros(): + cols = ['CO_GRUPO', 'TP_SEXO', 'NU_IDADE', 'NT_GER', 'NT_FG', 'NT_CE', + 'QE_I01','QE_I02','QE_I04','QE_I05','QE_I08'] + enade = pd.read_csv(arquivo, sep=';', decimal=',', usecols=cols) + enade = enade.loc[ + (enade.NU_IDADE > 20) & + (enade.NU_IDADE < 40) & + (enade.NT_GER > 0) + ] + enade.to_csv(data_path + 'enade_filtrado.csv', index=False) + +task_aplica_filtro = PythonOperator( + task_id="aplica_filtro", + python_callable=aplica_filtros, + dag=dag +) + +# Idade centralizada na média +# Idade centralizada ao quadrado + +def constroi_idade_centralizada(): + idade = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['NU_IDADE']) + idade['idadecent'] = idade.NU_IDADE - idade.NU_IDADE.mean() + idade[['idadecent']].to_csv(data_path + "idadecent.csv", index=False) + +def constroi_idade_cent_quad(): + idadecent = pd.read_csv(data_path + "idadecent.csv") + idadecent['idade2'] = idadecent.idadecent ** 2 + idadecent[['idade2']].to_csv(data_path + 'idadequadrado.csv', index=False) + + +task_idade_cent = PythonOperator( + task_id='constroi_idade_centralizada', + python_callable=constroi_idade_centralizada, + dag=dag +) + +task_idade_quad = PythonOperator( + task_id='constroi_idade_ao_quadrado', + python_callable=constroi_idade_cent_quad, + dag=dag +) + +def constroi_est_civil(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I01']) + filtro['estcivil'] = filtro.QE_I01.replace({ + 'A': 'Solteiro', + 'B': 'Casado', + 'C': 'Separado', + 'D': 'Viúvo', + 'E': 'Outro' + }) + filtro[['estcivil']].to_csv(data_path + 'estcivil.csv', index=False) + +task_est_civil = PythonOperator( + task_id='constroi_est_civil', + python_callable=constroi_est_civil, + dag=dag +) + +def constroi_cor(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I02']) + filtro['cor'] = filtro.QE_I02.replace({ + 'A': 'Branca', + 'B': 'Preta', + 'C': 'Amarela', + 'D': 'Parda', + 'E': 'Indígena', + 'F': "", + ' ': "" + }) + filtro[['cor']].to_csv(data_path + 'cor.csv', index=False) + +task_cor = PythonOperator( + task_id='constroi_cor_da_pele', + python_callable=constroi_cor, + dag=dag +) + + +def constroi_escopai(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I04']) + filtro['escopai'] = filtro.QE_I04.replace({ + 'A': 0, + 'B': 1, + 'C': 2, + 'D': 3, + 'E': 4, + 'F': 5 + }) + filtro[['escopai']].to_csv(data_path + 'escopai.csv', index=False) + +task_escopai = PythonOperator( + task_id='constroi_escopai', + python_callable=constroi_escopai, + dag=dag +) + +def constroi_escomae(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I05']) + filtro['escomae'] = filtro.QE_I05.replace({ + 'A': 0, + 'B': 1, + 'C': 2, + 'D': 3, + 'E': 4, + 'F': 5 + }) + filtro[['escomae']].to_csv(data_path + 'escomae.csv', index=False) + +task_escomae = PythonOperator( + task_id='constroi_escomae', + python_callable=constroi_escomae, + dag=dag +) + +def constroi_renda(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I08']) + filtro['renda'] = filtro.QE_I08.replace({ + 'A': 0, + 'B': 1, + 'C': 2, + 'D': 3, + 'E': 4, + 'F': 5, + 'G': 6 + }) + filtro[['renda']].to_csv(data_path + 'renda.csv', index=False) + +task_renda = PythonOperator( + task_id='constroi_renda', + python_callable=constroi_renda, + dag=dag +) + + +# Task de JOIN +def join_data(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv') + idadecent = pd.read_csv(data_path + 'idadecent.csv') + idadeaoquadrado = pd.read_csv(data_path + 'idadequadrado.csv') + estcivil = pd.read_csv(data_path + 'estcivil.csv') + cor = pd.read_csv(data_path + 'cor.csv') + escopai = pd.read_csv(data_path + 'escopai.csv') + escomae = pd.read_csv(data_path + 'escomae.csv') + renda = pd.read_csv(data_path + 'renda.csv') + + final = pd.concat([ + filtro, idadecent, idadeaoquadrado, estcivil, cor, + escopai, escomae, renda + ], + axis=1 + ) + + final.to_csv(data_path + 'enade_tratado.csv', index=False) + print(final) + +task_join = PythonOperator( + task_id='join_data', + python_callable=join_data, + dag=dag +) + +start_preprocessing >> get_data >> unzip_data >> task_aplica_filtro +task_aplica_filtro >> [task_idade_cent, task_est_civil, task_cor, + task_escopai, task_escomae, task_renda] + +task_idade_quad.set_upstream(task_idade_cent) + +task_join.set_upstream([ + task_est_civil, task_cor, task_idade_quad, + task_escopai, task_escomae, task_renda +]) \ No newline at end of file diff --git a/docker-compose-CeleryExecutor.yml b/docker-compose-CeleryExecutor.yml index af6ba37d..8ff0e9a1 100644 --- a/docker-compose-CeleryExecutor.yml +++ b/docker-compose-CeleryExecutor.yml @@ -16,7 +16,7 @@ services: # - ./pgdata:/var/lib/postgresql/data/pgdata webserver: - image: puckel/docker-airflow:2.0.0 + image: neylsoncrepalde/airflow-docker:latest restart: always depends_on: - postgres @@ -43,7 +43,7 @@ services: retries: 3 flower: - image: puckel/docker-airflow:2.0.0 + image: neylsoncrepalde/airflow-docker:latest restart: always depends_on: - redis @@ -55,7 +55,7 @@ services: command: celery flower scheduler: - image: puckel/docker-airflow:2.0.0 + image: neylsoncrepalde/airflow-docker:latest restart: always depends_on: - webserver @@ -74,7 +74,7 @@ services: command: scheduler worker: - image: puckel/docker-airflow:2.0.0 + image: neylsoncrepalde/airflow-docker:latest restart: always depends_on: - scheduler diff --git a/docker-compose-LocalExecutor.yml b/docker-compose-LocalExecutor.yml index 33311c00..78e88ab2 100644 --- a/docker-compose-LocalExecutor.yml +++ b/docker-compose-LocalExecutor.yml @@ -12,7 +12,7 @@ services: max-file: "3" webserver: - image: puckel/docker-airflow:2.0.0 + image: neylsoncrepalde/airflow-docker:latest restart: always depends_on: - postgres From a3ce36c6ce24eb6c8ee040cbe45c012e7b3befd0 Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Sat, 19 Dec 2020 22:00:48 -0300 Subject: [PATCH 09/20] mount data local volume --- .gitignore | 2 ++ docker-compose-CeleryExecutor.yml | 3 +++ docker-compose-LocalExecutor.yml | 1 + 3 files changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 991a0fb0..387bbbfd 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ sftp-config.json # Python __pycache__ + +.vscode/ \ No newline at end of file diff --git a/docker-compose-CeleryExecutor.yml b/docker-compose-CeleryExecutor.yml index 8ff0e9a1..64ec7423 100644 --- a/docker-compose-CeleryExecutor.yml +++ b/docker-compose-CeleryExecutor.yml @@ -31,6 +31,7 @@ services: # - REDIS_PASSWORD=redispass volumes: - ./dags:/usr/local/airflow/dags + - ./data:/usr/local/airflow/data # Uncomment to include custom plugins # - ./plugins:/usr/local/airflow/plugins ports: @@ -61,6 +62,7 @@ services: - webserver volumes: - ./dags:/usr/local/airflow/dags + - ./data:/usr/local/airflow/data # Uncomment to include custom plugins # - ./plugins:/usr/local/airflow/plugins environment: @@ -80,6 +82,7 @@ services: - scheduler volumes: - ./dags:/usr/local/airflow/dags + - ./data:/usr/local/airflow/data # Uncomment to include custom plugins # - ./plugins:/usr/local/airflow/plugins environment: diff --git a/docker-compose-LocalExecutor.yml b/docker-compose-LocalExecutor.yml index 78e88ab2..45466f7e 100644 --- a/docker-compose-LocalExecutor.yml +++ b/docker-compose-LocalExecutor.yml @@ -25,6 +25,7 @@ services: max-file: "3" volumes: - ./dags:/usr/local/airflow/dags + - ./data:/usr/local/airflow/data # - ./plugins:/usr/local/airflow/plugins ports: - "8080:8080" From 26b25b7f5db1f8941930c1d388b880ccd1d0ab89 Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Sat, 19 Dec 2020 22:12:21 -0300 Subject: [PATCH 10/20] correct enade folder --- dags/treino04.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/treino04.py b/dags/treino04.py index 5690e418..44668987 100644 --- a/dags/treino04.py +++ b/dags/treino04.py @@ -36,7 +36,7 @@ get_data = BashOperator( task_id="get-data", - bash_command='mkdir -p /usr/local/airflow/data && curl http://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', + bash_command='curl http://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', dag=dag ) From 8d5a1666867fe9c6d32f20a79c2c746c7ec80e45 Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Sat, 19 Dec 2020 22:18:58 -0300 Subject: [PATCH 11/20] correct enade url --- dags/treino04.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/treino04.py b/dags/treino04.py index 44668987..dbadd0f4 100644 --- a/dags/treino04.py +++ b/dags/treino04.py @@ -36,7 +36,7 @@ get_data = BashOperator( task_id="get-data", - bash_command='curl http://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', + bash_command='curl https://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', dag=dag ) From 2fe7ff5f32f4d9be8e2e054a58836ee5d7ce8ae0 Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Tue, 29 Dec 2020 11:41:06 -0300 Subject: [PATCH 12/20] change all image tags in readme to 2.0.0 --- README.md | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index f6d6cd23..ce8106c7 100644 --- a/README.md +++ b/README.md @@ -17,20 +17,21 @@ This repository contains **Dockerfile** of [apache-airflow](https://github.com/a ## Installation -Pull the image from the Docker repository. +Up to this moment, there is no public image `puckel/docker-airflow:2.0.0`, so we have to build it. After cloning this repo, you may do + + docker build -t puckel/docker-airflow:2.0.0 . - docker pull puckel/docker-airflow ## Build Optionally install [Extra Airflow Packages](https://airflow.incubator.apache.org/installation.html#extra-package) and/or python dependencies at build time : - docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" -t puckel/docker-airflow . - docker build --rm --build-arg PYTHON_DEPS="flask_oauthlib>=0.9" -t puckel/docker-airflow . + docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" -t puckel/docker-airflow:2.0.0 . + docker build --rm --build-arg PYTHON_DEPS="flask_oauthlib>=0.9" -t puckel/docker-airflow:2.0.0 . or combined - docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" --build-arg PYTHON_DEPS="flask_oauthlib>=0.9" -t puckel/docker-airflow . + docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" --build-arg PYTHON_DEPS="flask_oauthlib>=0.9" -t puckel/docker-airflow:2.0.0 . Don't forget to update the airflow images in the docker-compose files to puckel/docker-airflow:latest. @@ -38,7 +39,7 @@ Don't forget to update the airflow images in the docker-compose files to puckel/ By default, docker-airflow runs Airflow with **SequentialExecutor** : - docker run -d -p 8080:8080 puckel/docker-airflow webserver + docker run -d -p 8080:8080 puckel/docker-airflow:2.0.0 webserver If you want to run another executor, use the other docker-compose.yml files provided in this repository. @@ -54,7 +55,7 @@ NB : If you want to have DAGs example loaded (default=False), you've to set the `LOAD_EX=n` - docker run -d -p 8080:8080 -e LOAD_EX=y puckel/docker-airflow + docker run -d -p 8080:8080 -e LOAD_EX=y puckel/docker-airflow:2.0.0 If you want to use Ad hoc query, make sure you've configured connections: Go to Admin -> Connections and Edit "postgres_default" set this values (equivalent to values in airflow.cfg/docker-compose*.yml) : @@ -65,7 +66,7 @@ Go to Admin -> Connections and Edit "postgres_default" set this values (equivale For encrypted connection passwords (in Local or Celery Executor), you must have the same fernet_key. By default docker-airflow generates the fernet_key at startup, you have to set an environment variable in the docker-compose (ie: docker-compose-LocalExecutor.yml) file to set the same key accross containers. To generate a fernet_key : - docker run puckel/docker-airflow python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)" + docker run puckel/docker-airflow:2.0.0 python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)" ## Configuring Airflow @@ -114,7 +115,7 @@ This can be used to scale to a multi node setup using docker swarm. If you want to run other airflow sub-commands, such as `list_dags` or `clear` you can do so like this: - docker run --rm -ti puckel/docker-airflow airflow list_dags + docker run --rm -ti puckel/docker-airflow:2.0.0 airflow list_dags or with your docker-compose set up like this: @@ -122,8 +123,8 @@ or with your docker-compose set up like this: You can also use this to run a bash shell or any other command in the same environment that airflow would be run in: - docker run --rm -ti puckel/docker-airflow bash - docker run --rm -ti puckel/docker-airflow ipython + docker run --rm -ti puckel/docker-airflow:2.0.0 bash + docker run --rm -ti puckel/docker-airflow:2.0.0 ipython # Simplified SQL database configuration using PostgreSQL From 7a6874f48a68f556799f9842fba5da37cff5aecc Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Wed, 30 Dec 2020 11:29:48 -0300 Subject: [PATCH 13/20] corrige dags 04 e 05 --- .gitignore | 4 +- dags/treino04.py | 219 +++++++++-------------------------------------- dags/treino05.py | 207 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 249 insertions(+), 181 deletions(-) create mode 100644 dags/treino05.py diff --git a/.gitignore b/.gitignore index 387bbbfd..8b85f2f8 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,6 @@ sftp-config.json # Python __pycache__ -.vscode/ \ No newline at end of file +.vscode/ + +microdados_enade_2019* \ No newline at end of file diff --git a/dags/treino04.py b/dags/treino04.py index dbadd0f4..9384b31f 100644 --- a/dags/treino04.py +++ b/dags/treino04.py @@ -2,224 +2,83 @@ from airflow.operators.bash_operator import BashOperator from airflow.operators.python_operator import PythonOperator, BranchPythonOperator from datetime import datetime, timedelta -import os +import random import pandas as pd -import zipfile - -# Constantes -data_path = '/usr/local/airflow/data/microdados_enade_2019/2019/3.DADOS/' -arquivo = data_path + 'microdados_enade_2019.txt' default_args = { 'owner': 'Neylson Crepalde', "depends_on_past": False, - "start_date": datetime(2020, 11, 15, 17), + "start_date": datetime(2020, 11, 15, 1, 55), "email": ["airflow@airflow.com"], "email_on_failure": False, - "email_on_retry": False, - "retries": 1, - "retry_delay": timedelta(minutes=1), + "email_on_retry": False + #"retries": 1, + #"retry_delay": timedelta(minutes=1), } dag = DAG( "treino-04", - description="Paralelismos", + description="Uma dag com condicionais", default_args=default_args, - schedule_interval="*/10 * * * *" -) - -start_preprocessing = BashOperator( - task_id='start_preprocessing', - bash_command='echo "Start Preprocessing! Vai!"', - dag=dag + schedule_interval=timedelta(minutes=2) ) get_data = BashOperator( task_id="get-data", - bash_command='curl https://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', + bash_command='curl http://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', + trigger_rule="all_done", dag=dag ) + def unzip_file(): - with zipfile.ZipFile('/usr/local/airflow/data/microdados_enade_2019.zip', 'r') as zipped: - zipped.extractall('/usr/local/airflow/data') + with zipfile.ZipFile("/usr/local/airflow/data/microdados_enade_2019.zip", 'r') as zipped: + zipped.extractall("/usr/local/airflow/data") unzip_data = PythonOperator( - task_id='unzip_data', + task_id='unzip-data', python_callable=unzip_file, dag=dag ) -def aplica_filtros(): - cols = ['CO_GRUPO', 'TP_SEXO', 'NU_IDADE', 'NT_GER', 'NT_FG', 'NT_CE', - 'QE_I01','QE_I02','QE_I04','QE_I05','QE_I08'] - enade = pd.read_csv(arquivo, sep=';', decimal=',', usecols=cols) - enade = enade.loc[ - (enade.NU_IDADE > 20) & - (enade.NU_IDADE < 40) & - (enade.NT_GER > 0) - ] - enade.to_csv(data_path + 'enade_filtrado.csv', index=False) - -task_aplica_filtro = PythonOperator( - task_id="aplica_filtro", - python_callable=aplica_filtros, - dag=dag -) - -# Idade centralizada na média -# Idade centralizada ao quadrado - -def constroi_idade_centralizada(): - idade = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['NU_IDADE']) - idade['idadecent'] = idade.NU_IDADE - idade.NU_IDADE.mean() - idade[['idadecent']].to_csv(data_path + "idadecent.csv", index=False) - -def constroi_idade_cent_quad(): - idadecent = pd.read_csv(data_path + "idadecent.csv") - idadecent['idade2'] = idadecent.idadecent ** 2 - idadecent[['idade2']].to_csv(data_path + 'idadequadrado.csv', index=False) - - -task_idade_cent = PythonOperator( - task_id='constroi_idade_centralizada', - python_callable=constroi_idade_centralizada, - dag=dag -) - -task_idade_quad = PythonOperator( - task_id='constroi_idade_ao_quadrado', - python_callable=constroi_idade_cent_quad, - dag=dag -) - -def constroi_est_civil(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I01']) - filtro['estcivil'] = filtro.QE_I01.replace({ - 'A': 'Solteiro', - 'B': 'Casado', - 'C': 'Separado', - 'D': 'Viúvo', - 'E': 'Outro' - }) - filtro[['estcivil']].to_csv(data_path + 'estcivil.csv', index=False) - -task_est_civil = PythonOperator( - task_id='constroi_est_civil', - python_callable=constroi_est_civil, - dag=dag -) -def constroi_cor(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I02']) - filtro['cor'] = filtro.QE_I02.replace({ - 'A': 'Branca', - 'B': 'Preta', - 'C': 'Amarela', - 'D': 'Parda', - 'E': 'Indígena', - 'F': "", - ' ': "" - }) - filtro[['cor']].to_csv(data_path + 'cor.csv', index=False) +def select_student(): + df = pd.read_csv('/usr/local/airflow/data/microdados_enade_2019/2019/3.DADOS/microdados_enade_2019.txt', sep=';', decimal=',') + escolha = random.randint(0, df.shape[0]-1) + aluno = df.iloc[escolha] + return aluno.TP_SEXO -task_cor = PythonOperator( - task_id='constroi_cor_da_pele', - python_callable=constroi_cor, +pick_student = PythonOperator( + task_id="pick-student", + python_callable=select_student, dag=dag ) - -def constroi_escopai(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I04']) - filtro['escopai'] = filtro.QE_I04.replace({ - 'A': 0, - 'B': 1, - 'C': 2, - 'D': 3, - 'E': 4, - 'F': 5 - }) - filtro[['escopai']].to_csv(data_path + 'escopai.csv', index=False) - -task_escopai = PythonOperator( - task_id='constroi_escopai', - python_callable=constroi_escopai, +def MouF(**context): + value = context['task_instance'].xcom_pull(task_ids='pick-student') + if value == 'M': + return 'male_branch' + elif value == 'F': + return 'female_branch' + +male_of_female = BranchPythonOperator( + task_id='condition-male_or_female', + python_callable=MouF, + provide_context=True, dag=dag ) -def constroi_escomae(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I05']) - filtro['escomae'] = filtro.QE_I05.replace({ - 'A': 0, - 'B': 1, - 'C': 2, - 'D': 3, - 'E': 4, - 'F': 5 - }) - filtro[['escomae']].to_csv(data_path + 'escomae.csv', index=False) -task_escomae = PythonOperator( - task_id='constroi_escomae', - python_callable=constroi_escomae, +male_branch = BashOperator( + task_id="male_branch", + bash_command='echo "Estudante escolhido foi do sexo Masculino"', dag=dag ) -def constroi_renda(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I08']) - filtro['renda'] = filtro.QE_I08.replace({ - 'A': 0, - 'B': 1, - 'C': 2, - 'D': 3, - 'E': 4, - 'F': 5, - 'G': 6 - }) - filtro[['renda']].to_csv(data_path + 'renda.csv', index=False) - -task_renda = PythonOperator( - task_id='constroi_renda', - python_callable=constroi_renda, +female_branch = BashOperator( + task_id="female_branch", + bash_command='echo "Estudante escolhido foi do sexo Feminino"', dag=dag ) - -# Task de JOIN -def join_data(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv') - idadecent = pd.read_csv(data_path + 'idadecent.csv') - idadeaoquadrado = pd.read_csv(data_path + 'idadequadrado.csv') - estcivil = pd.read_csv(data_path + 'estcivil.csv') - cor = pd.read_csv(data_path + 'cor.csv') - escopai = pd.read_csv(data_path + 'escopai.csv') - escomae = pd.read_csv(data_path + 'escomae.csv') - renda = pd.read_csv(data_path + 'renda.csv') - - final = pd.concat([ - filtro, idadecent, idadeaoquadrado, estcivil, cor, - escopai, escomae, renda - ], - axis=1 - ) - - final.to_csv(data_path + 'enade_tratado.csv', index=False) - print(final) - -task_join = PythonOperator( - task_id='join_data', - python_callable=join_data, - dag=dag -) - -start_preprocessing >> get_data >> unzip_data >> task_aplica_filtro -task_aplica_filtro >> [task_idade_cent, task_est_civil, task_cor, - task_escopai, task_escomae, task_renda] - -task_idade_quad.set_upstream(task_idade_cent) - -task_join.set_upstream([ - task_est_civil, task_cor, task_idade_quad, - task_escopai, task_escomae, task_renda -]) \ No newline at end of file +get_data >> unzip_data >> pick_student >> male_of_female >> [male_branch, female_branch] \ No newline at end of file diff --git a/dags/treino05.py b/dags/treino05.py new file mode 100644 index 00000000..d97b8abc --- /dev/null +++ b/dags/treino05.py @@ -0,0 +1,207 @@ +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.python_operator import PythonOperator, BranchPythonOperator +from datetime import datetime, timedelta +import pandas as pd + +data_path = '/usr/local/airflow/data/microdados_enade_2019/2019/3.DADOS/' +arquivo = data_path + 'microdados_enade_2019.txt' + +default_args = { + 'owner': 'Neylson Crepalde', + "depends_on_past": False, + "start_date": datetime(2020, 11, 15, 1, 55), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False + #"retries": 1, + #"retry_delay": timedelta(minutes=1), +} + +dag = DAG( + "treino-05", + description="Uma dag com vários paralelismos", + default_args=default_args, + schedule_interval="*/10 * * * *" +) + +get_data = BashOperator( + task_id="get-data", + bash_command='curl http://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', + trigger_rule="all_done", + dag=dag +) + +def unzip_file(): + with zipfile.ZipFile("/usr/local/airflow/data/microdados_enade_2019.zip", 'r') as zipped: + zipped.extractall("/usr/local/airflow/data") + +unzip_data = PythonOperator( + task_id='unzip-data', + python_callable=unzip_file, + dag=dag +) + + +def aplica_filtros(): + cols = ['CO_GRUPO', 'TP_SEXO', 'NU_IDADE', 'NT_GER', 'NT_FG', 'NT_CE', + 'QE_I01','QE_I02','QE_I04','QE_I05','QE_I08'] + enade = pd.read_csv(arquivo, sep=';', decimal=',', usecols=cols) + enade = enade.loc[ + (enade.NU_IDADE > 20) & + (enade.NU_IDADE < 40) & + (enade.NT_GER > 0) + ] + enade.to_csv(data_path + 'enade_filtrado.csv', index=False) + +task_aplica_filtro = PythonOperator( + task_id="aplica_filtro", + python_callable=aplica_filtros, + dag=dag +) + +def constroi_idade_centralizada(): + idade = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['NU_IDADE']) + idade['idadecent'] = idade.NU_IDADE - idade.NU_IDADE.mean() + idade[['idadecent']].to_csv(data_path + "idadecent.csv", index=False) + +def constroi_idade_cent_quad(): + idadecent = pd.read_csv(data_path + "idadecent.csv") + idadecent['idade2'] = idadecent.idadecent ** 2 + idadecent[['idade2']].to_csv(data_path + 'idadequadrado.csv', index=False) + +task_idade_cent = PythonOperator( + task_id='constroi_idade_centralizada', + python_callable=constroi_idade_centralizada, + dag=dag +) + +task_idade_quad = PythonOperator( + task_id='constroi_idade_ao_quadrado', + python_callable=constroi_idade_cent_quad, + dag=dag +) + +def constroi_est_civil(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I01']) + filtro['estcivil'] = filtro.QE_I01.replace({ + 'A': 'Solteiro', + 'B': 'Casado', + 'C': 'Separado', + 'D': 'Viúvo', + 'E': 'Outro' + }) + filtro[['estcivil']].to_csv(data_path + 'estcivil.csv', index=False) + +task_est_civil = PythonOperator( + task_id='constroi_est_civil', + python_callable=constroi_est_civil, + dag=dag +) + +def constroi_cor(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I02']) + filtro['cor'] = filtro.QE_I02.replace({ + 'A': 'Branca', + 'B': 'Preta', + 'C': 'Amarela', + 'D': 'Parda', + 'E': 'Indígena', + 'F': "", + ' ': "" + }) + filtro[['cor']].to_csv(data_path + 'cor.csv', index=False) + +task_cor = PythonOperator( + task_id='constroi_cor_da_pele', + python_callable=constroi_cor, + dag=dag +) + +def constroi_escopai(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I04']) + filtro['escopai'] = filtro.QE_I04.replace({ + 'A': 0, + 'B': 1, + 'C': 2, + 'D': 3, + 'E': 4, + 'F': 5 + }) + filtro[['escopai']].to_csv(data_path + 'escopai.csv', index=False) + +task_escopai = PythonOperator( + task_id='constroi_escopai', + python_callable=constroi_escopai, + dag=dag +) + +def constroi_escomae(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I05']) + filtro['escomae'] = filtro.QE_I05.replace({ + 'A': 0, + 'B': 1, + 'C': 2, + 'D': 3, + 'E': 4, + 'F': 5 + }) + filtro[['escomae']].to_csv(data_path + 'escomae.csv', index=False) + +task_escomae = PythonOperator( + task_id='constroi_escomae', + python_callable=constroi_escomae, + dag=dag +) + +def constroi_renda(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I08']) + filtro['renda'] = filtro.QE_I08.replace({ + 'A': 0, + 'B': 1, + 'C': 2, + 'D': 3, + 'E': 4, + 'F': 5, + 'G': 6 + }) + filtro[['renda']].to_csv(data_path + 'renda.csv', index=False) + +task_renda = PythonOperator( + task_id='constroi_renda', + python_callable=constroi_renda, + dag=dag +) + +def join_data(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv') + idadecent = pd.read_csv(data_path + 'idadecent.csv') + idadequadrado = pd.read_csv(data_path + 'idadequadrado.csv') + estcivil = pd.read_csv(data_path + 'estcivil.csv') + cor = pd.read_csv(data_path + 'cor.csv') + escopai = pd.read_csv(data_path + 'escopai.csv') + escomae = pd.read_csv(data_path + 'escomae.csv') + renda = pd.read_csv(data_path + 'renda.csv') + + final = pd.concat([filtro, idadecent, idadequadrado, estcivil, cor, + escopai, escomae, renda], axis=1) + final.to_csv(data_path + 'enade_tratado.csv', index=False) + print(final) + +task_join = PythonOperator( + task_id='join_data', + python_callable=join_data, + dag=dag +) + + + +get_data >> unzip_data >> task_aplica_filtro >> [ + task_idade_cent, task_est_civil, task_cor ,task_escopai, task_escomae, task_renda +] +task_idade_quad.set_upstream(task_idade_cent) +task_join.set_upstream([ + task_idade_cent, task_est_civil, task_escopai, task_cor, task_escomae, task_renda, task_idade_quad +]) + + From 7fc8c2fbada72cdf782f34fe41fe55fb5ca75dc1 Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Wed, 30 Dec 2020 11:31:06 -0300 Subject: [PATCH 14/20] corrige tag de imagem --- docker-compose-CeleryExecutor.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose-CeleryExecutor.yml b/docker-compose-CeleryExecutor.yml index 64ec7423..11af8f9f 100644 --- a/docker-compose-CeleryExecutor.yml +++ b/docker-compose-CeleryExecutor.yml @@ -16,7 +16,7 @@ services: # - ./pgdata:/var/lib/postgresql/data/pgdata webserver: - image: neylsoncrepalde/airflow-docker:latest + image: neylsoncrepalde/airflow-docker:2.0.0 restart: always depends_on: - postgres @@ -44,7 +44,7 @@ services: retries: 3 flower: - image: neylsoncrepalde/airflow-docker:latest + image: neylsoncrepalde/airflow-docker:2.0.0 restart: always depends_on: - redis @@ -56,7 +56,7 @@ services: command: celery flower scheduler: - image: neylsoncrepalde/airflow-docker:latest + image: neylsoncrepalde/airflow-docker:2.0.0 restart: always depends_on: - webserver @@ -76,7 +76,7 @@ services: command: scheduler worker: - image: neylsoncrepalde/airflow-docker:latest + image: neylsoncrepalde/airflow-docker:2.0.0 restart: always depends_on: - scheduler From b5f11fda0c6f4ef1364917541c4799deb941ab8b Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Wed, 30 Dec 2020 12:02:26 -0300 Subject: [PATCH 15/20] corrige fonte de enade --- dags/treino04.py | 5 +++-- dags/treino05.py | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/dags/treino04.py b/dags/treino04.py index 9384b31f..03d37a7d 100644 --- a/dags/treino04.py +++ b/dags/treino04.py @@ -2,13 +2,14 @@ from airflow.operators.bash_operator import BashOperator from airflow.operators.python_operator import PythonOperator, BranchPythonOperator from datetime import datetime, timedelta +import zipfile import random import pandas as pd default_args = { 'owner': 'Neylson Crepalde', "depends_on_past": False, - "start_date": datetime(2020, 11, 15, 1, 55), + "start_date": datetime(2020, 12, 30, 14, 50), "email": ["airflow@airflow.com"], "email_on_failure": False, "email_on_retry": False @@ -25,7 +26,7 @@ get_data = BashOperator( task_id="get-data", - bash_command='curl http://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', + bash_command='curl https://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', trigger_rule="all_done", dag=dag ) diff --git a/dags/treino05.py b/dags/treino05.py index d97b8abc..776a7aa4 100644 --- a/dags/treino05.py +++ b/dags/treino05.py @@ -2,6 +2,7 @@ from airflow.operators.bash_operator import BashOperator from airflow.operators.python_operator import PythonOperator, BranchPythonOperator from datetime import datetime, timedelta +import zipfile import pandas as pd data_path = '/usr/local/airflow/data/microdados_enade_2019/2019/3.DADOS/' @@ -10,7 +11,7 @@ default_args = { 'owner': 'Neylson Crepalde', "depends_on_past": False, - "start_date": datetime(2020, 11, 15, 1, 55), + "start_date": datetime(2020, 12, 30, 14, 50), "email": ["airflow@airflow.com"], "email_on_failure": False, "email_on_retry": False @@ -22,12 +23,12 @@ "treino-05", description="Uma dag com vários paralelismos", default_args=default_args, - schedule_interval="*/10 * * * *" + schedule_interval="*/3 * * * *" ) get_data = BashOperator( task_id="get-data", - bash_command='curl http://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', + bash_command='curl https://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', trigger_rule="all_done", dag=dag ) From 44ee88b0673c31e58766cc458fa85a5f507d055a Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Wed, 30 Dec 2020 12:12:31 -0300 Subject: [PATCH 16/20] =?UTF-8?q?corrige=20hor=C3=A1rio=20de=20in=C3=ADcio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/treino04.py | 2 +- dags/treino05.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dags/treino04.py b/dags/treino04.py index 03d37a7d..f7780d9b 100644 --- a/dags/treino04.py +++ b/dags/treino04.py @@ -9,7 +9,7 @@ default_args = { 'owner': 'Neylson Crepalde', "depends_on_past": False, - "start_date": datetime(2020, 12, 30, 14, 50), + "start_date": datetime(2020, 12, 30, 18, 10), "email": ["airflow@airflow.com"], "email_on_failure": False, "email_on_retry": False diff --git a/dags/treino05.py b/dags/treino05.py index 776a7aa4..7636e6fc 100644 --- a/dags/treino05.py +++ b/dags/treino05.py @@ -11,7 +11,7 @@ default_args = { 'owner': 'Neylson Crepalde', "depends_on_past": False, - "start_date": datetime(2020, 12, 30, 14, 50), + "start_date": datetime(2020, 12, 30, 18, 10), "email": ["airflow@airflow.com"], "email_on_failure": False, "email_on_retry": False From 405dd20941c977f4c8d9f42e3c6041a93066860c Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Wed, 30 Dec 2020 12:32:01 -0300 Subject: [PATCH 17/20] Create .gitkeep --- data/.gitkeep | 1 + 1 file changed, 1 insertion(+) create mode 100644 data/.gitkeep diff --git a/data/.gitkeep b/data/.gitkeep new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/data/.gitkeep @@ -0,0 +1 @@ + From aac10b109367d288a8cbf0658a7c909a5a4231ce Mon Sep 17 00:00:00 2001 From: Neylson Crepalde Date: Sun, 31 Jan 2021 17:13:19 -0300 Subject: [PATCH 18/20] add requirements to image --- .gitignore | 4 +++- Dockerfile | 1 + docker-compose-CeleryExecutor.yml | 8 ++++---- requirements.txt | 3 +++ 4 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 8b85f2f8..f2e6dde3 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,6 @@ __pycache__ .vscode/ -microdados_enade_2019* \ No newline at end of file +microdados_enade_2019* + +igti_bootcamp* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index b15d94d8..eaafe5e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -75,6 +75,7 @@ RUN set -ex \ COPY script/entrypoint.sh /entrypoint.sh COPY config/airflow.cfg ${AIRFLOW_USER_HOME}/airflow.cfg +COPY requirements.txt ${AIRFLOW_USER_HOME}/requirements.txt RUN chown -R airflow: ${AIRFLOW_USER_HOME} diff --git a/docker-compose-CeleryExecutor.yml b/docker-compose-CeleryExecutor.yml index 11af8f9f..64ec7423 100644 --- a/docker-compose-CeleryExecutor.yml +++ b/docker-compose-CeleryExecutor.yml @@ -16,7 +16,7 @@ services: # - ./pgdata:/var/lib/postgresql/data/pgdata webserver: - image: neylsoncrepalde/airflow-docker:2.0.0 + image: neylsoncrepalde/airflow-docker:latest restart: always depends_on: - postgres @@ -44,7 +44,7 @@ services: retries: 3 flower: - image: neylsoncrepalde/airflow-docker:2.0.0 + image: neylsoncrepalde/airflow-docker:latest restart: always depends_on: - redis @@ -56,7 +56,7 @@ services: command: celery flower scheduler: - image: neylsoncrepalde/airflow-docker:2.0.0 + image: neylsoncrepalde/airflow-docker:latest restart: always depends_on: - webserver @@ -76,7 +76,7 @@ services: command: scheduler worker: - image: neylsoncrepalde/airflow-docker:2.0.0 + image: neylsoncrepalde/airflow-docker:latest restart: always depends_on: - scheduler diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..4ce062b4 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +boto3 +pandas +pymongo From 7f167844ddb58ce31381a8f4711c83ad2079be29 Mon Sep 17 00:00:00 2001 From: Higor Carneiro Passos Date: Tue, 30 Nov 2021 14:55:40 -0300 Subject: [PATCH 19/20] Changed volume settings on celery and local executor docker-compose file and updating the scale parameter (deprecated) --- .dockerignore | 2 +- .github/workflows/ci.yml | 36 +- .gitignore | 69 +- LICENSE | 402 +++--- README.md | 354 ++--- config/airflow.cfg | 2000 ++++++++++++++--------------- dags/{treino01.py => t01.py} | 80 +- dags/treino02.py | 104 +- dags/treino03.py | 158 +-- dags/treino04.py | 168 +-- dags/treino05.py | 416 +++--- dags/tuto.py | 96 +- data/.gitkeep | 2 +- docker-compose-CeleryExecutor.yml | 193 +-- docker-compose-LocalExecutor.yml | 75 +- requirements.txt | 9 +- script/entrypoint.sh | 272 ++-- 17 files changed, 2224 insertions(+), 2212 deletions(-) rename dags/{treino01.py => t01.py} (95%) diff --git a/.dockerignore b/.dockerignore index 6b8710a7..46f8b9aa 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1 @@ -.git +.git diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a4f851c..0079ba60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,18 +1,18 @@ -name: CI - -on: - push: - branches: - - master - pull_request: - branches: - - master - -jobs: - ci: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - run: docker build -t "${PWD##*/}" . - - run: docker run "${PWD##*/}" python -V - - run: docker run "${PWD##*/}" version +name: CI + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + ci: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - run: docker build -t "${PWD##*/}" . + - run: docker run "${PWD##*/}" python -V + - run: docker run "${PWD##*/}" version diff --git a/.gitignore b/.gitignore index f2e6dde3..c4fc2162 100644 --- a/.gitignore +++ b/.gitignore @@ -1,32 +1,37 @@ -### Vim ### -[._]*.s[a-w][a-z] -[._]s[a-w][a-z] -*.un~ -Session.vim -.netrwhist -*~ - -### SublimeText ### -# cache files for sublime text -*.tmlanguage.cache -*.tmPreferences.cache -*.stTheme.cache - -# workspace files are user-specific -*.sublime-workspace - -# project files should be checked into the repository, unless a significant -# proportion of contributors will probably not be using SublimeText -# *.sublime-project - -# sftp configuration file -sftp-config.json - -# Python -__pycache__ - -.vscode/ - -microdados_enade_2019* - -igti_bootcamp* \ No newline at end of file +### Vim ### +[._]*.s[a-w][a-z] +[._]s[a-w][a-z] +*.un~ +Session.vim +.netrwhist +*~ + +### SublimeText ### +# cache files for sublime text +*.tmlanguage.cache +*.tmPreferences.cache +*.stTheme.cache + +# workspace files are user-specific +*.sublime-workspace + +# project files should be checked into the repository, unless a significant +# proportion of contributors will probably not be using SublimeText +# *.sublime-project + +# sftp configuration file +sftp-config.json + +# Python +__pycache__ + +.vscode/ + +microdados_enade_2019* + +igti_bootcamp* +treino02.py +treino03.py +treino04.py +treino05.py +tuto.py \ No newline at end of file diff --git a/LICENSE b/LICENSE index 917c8efe..18b434a8 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,201 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2017 Matthieu "Puckel_" Roisil - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 Matthieu "Puckel_" Roisil + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index ce8106c7..3f9e12ce 100644 --- a/README.md +++ b/README.md @@ -1,177 +1,177 @@ -# docker-airflow -[![CI status](https://github.com/puckel/docker-airflow/workflows/CI/badge.svg?branch=master)](https://github.com/puckel/docker-airflow/actions?query=workflow%3ACI+branch%3Amaster+event%3Apush) -[![Docker Build status](https://img.shields.io/docker/build/puckel/docker-airflow?style=plastic)](https://hub.docker.com/r/puckel/docker-airflow/tags?ordering=last_updated) - -[![Docker Hub](https://img.shields.io/badge/docker-ready-blue.svg)](https://hub.docker.com/r/puckel/docker-airflow/) -[![Docker Pulls](https://img.shields.io/docker/pulls/puckel/docker-airflow.svg)]() -[![Docker Stars](https://img.shields.io/docker/stars/puckel/docker-airflow.svg)]() - -This repository contains **Dockerfile** of [apache-airflow](https://github.com/apache/incubator-airflow) for [Docker](https://www.docker.com/)'s [automated build](https://registry.hub.docker.com/u/puckel/docker-airflow/) published to the public [Docker Hub Registry](https://registry.hub.docker.com/). - -## Informations - -* Based on Python (3.8-slim-buster) official Image [python:3.8-slim-buster](https://hub.docker.com/_/python/) and uses the official [Postgres](https://hub.docker.com/_/postgres/) as backend and [Redis](https://hub.docker.com/_/redis/) as queue -* Install [Docker](https://www.docker.com/) -* Install [Docker Compose](https://docs.docker.com/compose/install/) -* Following the Airflow release from [Python Package Index](https://pypi.python.org/pypi/apache-airflow) - -## Installation - -Up to this moment, there is no public image `puckel/docker-airflow:2.0.0`, so we have to build it. After cloning this repo, you may do - - docker build -t puckel/docker-airflow:2.0.0 . - - -## Build - -Optionally install [Extra Airflow Packages](https://airflow.incubator.apache.org/installation.html#extra-package) and/or python dependencies at build time : - - docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" -t puckel/docker-airflow:2.0.0 . - docker build --rm --build-arg PYTHON_DEPS="flask_oauthlib>=0.9" -t puckel/docker-airflow:2.0.0 . - -or combined - - docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" --build-arg PYTHON_DEPS="flask_oauthlib>=0.9" -t puckel/docker-airflow:2.0.0 . - -Don't forget to update the airflow images in the docker-compose files to puckel/docker-airflow:latest. - -## Usage - -By default, docker-airflow runs Airflow with **SequentialExecutor** : - - docker run -d -p 8080:8080 puckel/docker-airflow:2.0.0 webserver - -If you want to run another executor, use the other docker-compose.yml files provided in this repository. - -For **LocalExecutor** : - - docker-compose -f docker-compose-LocalExecutor.yml up -d - -For **CeleryExecutor** : - - docker-compose -f docker-compose-CeleryExecutor.yml up -d - -NB : If you want to have DAGs example loaded (default=False), you've to set the following environment variable : - -`LOAD_EX=n` - - docker run -d -p 8080:8080 -e LOAD_EX=y puckel/docker-airflow:2.0.0 - -If you want to use Ad hoc query, make sure you've configured connections: -Go to Admin -> Connections and Edit "postgres_default" set this values (equivalent to values in airflow.cfg/docker-compose*.yml) : -- Host : postgres -- Schema : airflow -- Login : airflow -- Password : airflow - -For encrypted connection passwords (in Local or Celery Executor), you must have the same fernet_key. By default docker-airflow generates the fernet_key at startup, you have to set an environment variable in the docker-compose (ie: docker-compose-LocalExecutor.yml) file to set the same key accross containers. To generate a fernet_key : - - docker run puckel/docker-airflow:2.0.0 python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)" - -## Configuring Airflow - -It's possible to set any configuration value for Airflow from environment variables, which are used over values from the airflow.cfg. - -The general rule is the environment variable should be named `AIRFLOW__
__`, for example `AIRFLOW__CORE__SQL_ALCHEMY_CONN` sets the `sql_alchemy_conn` config option in the `[core]` section. - -Check out the [Airflow documentation](http://airflow.readthedocs.io/en/latest/howto/set-config.html#setting-configuration-options) for more details - -You can also define connections via environment variables by prefixing them with `AIRFLOW_CONN_` - for example `AIRFLOW_CONN_POSTGRES_MASTER=postgres://user:password@localhost:5432/master` for a connection called "postgres_master". The value is parsed as a URI. This will work for hooks etc, but won't show up in the "Ad-hoc Query" section unless an (empty) connection is also created in the DB - -## Custom Airflow plugins - -Airflow allows for custom user-created plugins which are typically found in `${AIRFLOW_HOME}/plugins` folder. Documentation on plugins can be found [here](https://airflow.apache.org/plugins.html) - -In order to incorporate plugins into your docker container -- Create the plugins folders `plugins/` with your custom plugins. -- Mount the folder as a volume by doing either of the following: - - Include the folder as a volume in command-line `-v $(pwd)/plugins/:/usr/local/airflow/plugins` - - Use docker-compose-LocalExecutor.yml or docker-compose-CeleryExecutor.yml which contains support for adding the plugins folder as a volume - -## Install custom python package - -- Create a file "requirements.txt" with the desired python modules -- Mount this file as a volume `-v $(pwd)/requirements.txt:/requirements.txt` (or add it as a volume in docker-compose file) -- The entrypoint.sh script execute the pip install command (with --user option) - -## UI Links - -- Airflow: [localhost:8080](http://localhost:8080/) -- Flower: [localhost:5555](http://localhost:5555/) - -To log into airflow webserver, the default credentials are -- username: airflow -- password: airflow - -## Scale the number of workers - -Easy scaling using docker-compose: - - docker-compose -f docker-compose-CeleryExecutor.yml scale worker=5 - -This can be used to scale to a multi node setup using docker swarm. - -## Running other airflow commands - -If you want to run other airflow sub-commands, such as `list_dags` or `clear` you can do so like this: - - docker run --rm -ti puckel/docker-airflow:2.0.0 airflow list_dags - -or with your docker-compose set up like this: - - docker-compose -f docker-compose-CeleryExecutor.yml run --rm webserver airflow list_dags - -You can also use this to run a bash shell or any other command in the same environment that airflow would be run in: - - docker run --rm -ti puckel/docker-airflow:2.0.0 bash - docker run --rm -ti puckel/docker-airflow:2.0.0 ipython - -# Simplified SQL database configuration using PostgreSQL - -If the executor type is set to anything else than *SequentialExecutor* you'll need an SQL database. -Here is a list of PostgreSQL configuration variables and their default values. They're used to compute -the `AIRFLOW__CORE__SQL_ALCHEMY_CONN` and `AIRFLOW__CELERY__RESULT_BACKEND` variables when needed for you -if you don't provide them explicitly: - -| Variable | Default value | Role | -|---------------------|---------------|----------------------| -| `POSTGRES_HOST` | `postgres` | Database server host | -| `POSTGRES_PORT` | `5432` | Database server port | -| `POSTGRES_USER` | `airflow` | Database user | -| `POSTGRES_PASSWORD` | `airflow` | Database password | -| `POSTGRES_DB` | `airflow` | Database name | -| `POSTGRES_EXTRAS` | empty | Extras parameters | - -You can also use those variables to adapt your compose file to match an existing PostgreSQL instance managed elsewhere. - -Please refer to the Airflow documentation to understand the use of extras parameters, for example in order to configure -a connection that uses TLS encryption. - -Here's an important thing to consider: - -> When specifying the connection as URI (in AIRFLOW_CONN_* variable) you should specify it following the standard syntax of DB connections, -> where extras are passed as parameters of the URI (note that all components of the URI should be URL-encoded). - -Therefore you must provide extras parameters URL-encoded, starting with a leading `?`. For example: - - POSTGRES_EXTRAS="?sslmode=verify-full&sslrootcert=%2Fetc%2Fssl%2Fcerts%2Fca-certificates.crt" - -# Simplified Celery broker configuration using Redis - -If the executor type is set to *CeleryExecutor* you'll need a Celery broker. Here is a list of Redis configuration variables -and their default values. They're used to compute the `AIRFLOW__CELERY__BROKER_URL` variable for you if you don't provide -it explicitly: - -| Variable | Default value | Role | -|-------------------|---------------|--------------------------------| -| `REDIS_PROTO` | `redis://` | Protocol | -| `REDIS_HOST` | `redis` | Redis server host | -| `REDIS_PORT` | `6379` | Redis server port | -| `REDIS_PASSWORD` | empty | If Redis is password protected | -| `REDIS_DBNUM` | `1` | Database number | - -You can also use those variables to adapt your compose file to match an existing Redis instance managed elsewhere. - -# Wanna help? - -Fork, improve and PR. +# docker-airflow +[![CI status](https://github.com/puckel/docker-airflow/workflows/CI/badge.svg?branch=master)](https://github.com/puckel/docker-airflow/actions?query=workflow%3ACI+branch%3Amaster+event%3Apush) +[![Docker Build status](https://img.shields.io/docker/build/puckel/docker-airflow?style=plastic)](https://hub.docker.com/r/puckel/docker-airflow/tags?ordering=last_updated) + +[![Docker Hub](https://img.shields.io/badge/docker-ready-blue.svg)](https://hub.docker.com/r/puckel/docker-airflow/) +[![Docker Pulls](https://img.shields.io/docker/pulls/puckel/docker-airflow.svg)]() +[![Docker Stars](https://img.shields.io/docker/stars/puckel/docker-airflow.svg)]() + +This repository contains **Dockerfile** of [apache-airflow](https://github.com/apache/incubator-airflow) for [Docker](https://www.docker.com/)'s [automated build](https://registry.hub.docker.com/u/puckel/docker-airflow/) published to the public [Docker Hub Registry](https://registry.hub.docker.com/). + +## Informations + +* Based on Python (3.8-slim-buster) official Image [python:3.8-slim-buster](https://hub.docker.com/_/python/) and uses the official [Postgres](https://hub.docker.com/_/postgres/) as backend and [Redis](https://hub.docker.com/_/redis/) as queue +* Install [Docker](https://www.docker.com/) +* Install [Docker Compose](https://docs.docker.com/compose/install/) +* Following the Airflow release from [Python Package Index](https://pypi.python.org/pypi/apache-airflow) + +## Installation + +Up to this moment, there is no public image `puckel/docker-airflow:2.0.0`, so we have to build it. After cloning this repo, you may do + + docker build -t puckel/docker-airflow:2.0.0 . + + +## Build + +Optionally install [Extra Airflow Packages](https://airflow.incubator.apache.org/installation.html#extra-package) and/or python dependencies at build time : + + docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" -t puckel/docker-airflow:2.0.0 . + docker build --rm --build-arg PYTHON_DEPS="flask_oauthlib>=0.9" -t puckel/docker-airflow:2.0.0 . + +or combined + + docker build --rm --build-arg AIRFLOW_DEPS="datadog,dask" --build-arg PYTHON_DEPS="flask_oauthlib>=0.9" -t puckel/docker-airflow:2.0.0 . + +Don't forget to update the airflow images in the docker-compose files to puckel/docker-airflow:latest. + +## Usage + +By default, docker-airflow runs Airflow with **SequentialExecutor** : + + docker run -d -p 8080:8080 puckel/docker-airflow:2.0.0 webserver + +If you want to run another executor, use the other docker-compose.yml files provided in this repository. + +For **LocalExecutor** : + + docker-compose -f docker-compose-LocalExecutor.yml up -d + +For **CeleryExecutor** : + + docker-compose -f docker-compose-CeleryExecutor.yml up -d + +NB : If you want to have DAGs example loaded (default=False), you've to set the following environment variable : + +`LOAD_EX=n` + + docker run -d -p 8080:8080 -e LOAD_EX=y puckel/docker-airflow:2.0.0 + +If you want to use Ad hoc query, make sure you've configured connections: +Go to Admin -> Connections and Edit "postgres_default" set this values (equivalent to values in airflow.cfg/docker-compose*.yml) : +- Host : postgres +- Schema : airflow +- Login : airflow +- Password : airflow + +For encrypted connection passwords (in Local or Celery Executor), you must have the same fernet_key. By default docker-airflow generates the fernet_key at startup, you have to set an environment variable in the docker-compose (ie: docker-compose-LocalExecutor.yml) file to set the same key accross containers. To generate a fernet_key : + + docker run puckel/docker-airflow:2.0.0 python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)" + +## Configuring Airflow + +It's possible to set any configuration value for Airflow from environment variables, which are used over values from the airflow.cfg. + +The general rule is the environment variable should be named `AIRFLOW__
__`, for example `AIRFLOW__CORE__SQL_ALCHEMY_CONN` sets the `sql_alchemy_conn` config option in the `[core]` section. + +Check out the [Airflow documentation](http://airflow.readthedocs.io/en/latest/howto/set-config.html#setting-configuration-options) for more details + +You can also define connections via environment variables by prefixing them with `AIRFLOW_CONN_` - for example `AIRFLOW_CONN_POSTGRES_MASTER=postgres://user:password@localhost:5432/master` for a connection called "postgres_master". The value is parsed as a URI. This will work for hooks etc, but won't show up in the "Ad-hoc Query" section unless an (empty) connection is also created in the DB + +## Custom Airflow plugins + +Airflow allows for custom user-created plugins which are typically found in `${AIRFLOW_HOME}/plugins` folder. Documentation on plugins can be found [here](https://airflow.apache.org/plugins.html) + +In order to incorporate plugins into your docker container +- Create the plugins folders `plugins/` with your custom plugins. +- Mount the folder as a volume by doing either of the following: + - Include the folder as a volume in command-line `-v $(pwd)/plugins/:/usr/local/airflow/plugins` + - Use docker-compose-LocalExecutor.yml or docker-compose-CeleryExecutor.yml which contains support for adding the plugins folder as a volume + +## Install custom python package + +- Create a file "requirements.txt" with the desired python modules +- Mount this file as a volume `-v $(pwd)/requirements.txt:/requirements.txt` (or add it as a volume in docker-compose file) +- The entrypoint.sh script execute the pip install command (with --user option) + +## UI Links + +- Airflow: [localhost:8080](http://localhost:8080/) +- Flower: [localhost:5555](http://localhost:5555/) + +To log into airflow webserver, the default credentials are +- username: airflow +- password: airflow + +## Scale the number of workers + +Easy scaling using docker-compose: + + docker-compose -f docker-compose-CeleryExecutor.yml up -d --scale worker=5 + +This can be used to scale to a multi node setup using docker swarm. + +## Running other airflow commands + +If you want to run other airflow sub-commands, such as `list_dags` or `clear` you can do so like this: + + docker run --rm -ti puckel/docker-airflow:2.0.0 airflow list_dags + +or with your docker-compose set up like this: + + docker-compose -f docker-compose-CeleryExecutor.yml run --rm webserver airflow list_dags + +You can also use this to run a bash shell or any other command in the same environment that airflow would be run in: + + docker run --rm -ti puckel/docker-airflow:2.0.0 bash + docker run --rm -ti puckel/docker-airflow:2.0.0 ipython + +# Simplified SQL database configuration using PostgreSQL + +If the executor type is set to anything else than *SequentialExecutor* you'll need an SQL database. +Here is a list of PostgreSQL configuration variables and their default values. They're used to compute +the `AIRFLOW__CORE__SQL_ALCHEMY_CONN` and `AIRFLOW__CELERY__RESULT_BACKEND` variables when needed for you +if you don't provide them explicitly: + +| Variable | Default value | Role | +|---------------------|---------------|----------------------| +| `POSTGRES_HOST` | `postgres` | Database server host | +| `POSTGRES_PORT` | `5432` | Database server port | +| `POSTGRES_USER` | `airflow` | Database user | +| `POSTGRES_PASSWORD` | `airflow` | Database password | +| `POSTGRES_DB` | `airflow` | Database name | +| `POSTGRES_EXTRAS` | empty | Extras parameters | + +You can also use those variables to adapt your compose file to match an existing PostgreSQL instance managed elsewhere. + +Please refer to the Airflow documentation to understand the use of extras parameters, for example in order to configure +a connection that uses TLS encryption. + +Here's an important thing to consider: + +> When specifying the connection as URI (in AIRFLOW_CONN_* variable) you should specify it following the standard syntax of DB connections, +> where extras are passed as parameters of the URI (note that all components of the URI should be URL-encoded). + +Therefore you must provide extras parameters URL-encoded, starting with a leading `?`. For example: + + POSTGRES_EXTRAS="?sslmode=verify-full&sslrootcert=%2Fetc%2Fssl%2Fcerts%2Fca-certificates.crt" + +# Simplified Celery broker configuration using Redis + +If the executor type is set to *CeleryExecutor* you'll need a Celery broker. Here is a list of Redis configuration variables +and their default values. They're used to compute the `AIRFLOW__CELERY__BROKER_URL` variable for you if you don't provide +it explicitly: + +| Variable | Default value | Role | +|-------------------|---------------|--------------------------------| +| `REDIS_PROTO` | `redis://` | Protocol | +| `REDIS_HOST` | `redis` | Redis server host | +| `REDIS_PORT` | `6379` | Redis server port | +| `REDIS_PASSWORD` | empty | If Redis is password protected | +| `REDIS_DBNUM` | `1` | Database number | + +You can also use those variables to adapt your compose file to match an existing Redis instance managed elsewhere. + +# Wanna help? + +Fork, improve and PR. diff --git a/config/airflow.cfg b/config/airflow.cfg index a7dfed82..b2ada9d1 100644 --- a/config/airflow.cfg +++ b/config/airflow.cfg @@ -1,1000 +1,1000 @@ -[core] -# The folder where your airflow pipelines live, most likely a -# subfolder in a code repository. This path must be absolute. -dags_folder = /usr/local/airflow/dags - -# Hostname by providing a path to a callable, which will resolve the hostname. -# The format is "package.function". -# -# For example, default value "socket.getfqdn" means that result from getfqdn() of "socket" -# package will be used as hostname. -# -# No argument should be required in the function specified. -# If using IP address as hostname is preferred, use value ``airflow.utils.net.get_host_ip_address`` -hostname_callable = socket.getfqdn - -# Default timezone in case supplied date times are naive -# can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam) -default_timezone = utc - -# The executor class that airflow should use. Choices include -# ``SequentialExecutor``, ``LocalExecutor``, ``CeleryExecutor``, ``DaskExecutor``, -# ``KubernetesExecutor``, ``CeleryKubernetesExecutor`` or the -# full import path to the class when using a custom executor. -executor = SequentialExecutor - -# The SqlAlchemy connection string to the metadata database. -# SqlAlchemy supports many different database engine, more information -# their website -sql_alchemy_conn = sqlite:////usr/local/airflow/airflow.db - -# The encoding for the databases -sql_engine_encoding = utf-8 - -# Collation for ``dag_id``, ``task_id``, ``key`` columns in case they have different encoding. -# This is particularly useful in case of mysql with utf8mb4 encoding because -# primary keys for XCom table has too big size and ``sql_engine_collation_for_ids`` should -# be set to ``utf8mb3_general_ci``. -# sql_engine_collation_for_ids = - -# If SqlAlchemy should pool database connections. -sql_alchemy_pool_enabled = True - -# The SqlAlchemy pool size is the maximum number of database connections -# in the pool. 0 indicates no limit. -sql_alchemy_pool_size = 5 - -# The maximum overflow size of the pool. -# When the number of checked-out connections reaches the size set in pool_size, -# additional connections will be returned up to this limit. -# When those additional connections are returned to the pool, they are disconnected and discarded. -# It follows then that the total number of simultaneous connections the pool will allow -# is pool_size + max_overflow, -# and the total number of "sleeping" connections the pool will allow is pool_size. -# max_overflow can be set to ``-1`` to indicate no overflow limit; -# no limit will be placed on the total number of concurrent connections. Defaults to ``10``. -sql_alchemy_max_overflow = 10 - -# The SqlAlchemy pool recycle is the number of seconds a connection -# can be idle in the pool before it is invalidated. This config does -# not apply to sqlite. If the number of DB connections is ever exceeded, -# a lower config value will allow the system to recover faster. -sql_alchemy_pool_recycle = 1800 - -# Check connection at the start of each connection pool checkout. -# Typically, this is a simple statement like "SELECT 1". -# More information here: -# https://docs.sqlalchemy.org/en/13/core/pooling.html#disconnect-handling-pessimistic -sql_alchemy_pool_pre_ping = True - -# The schema to use for the metadata database. -# SqlAlchemy supports databases with the concept of multiple schemas. -sql_alchemy_schema = - -# Import path for connect args in SqlAlchemy. Defaults to an empty dict. -# This is useful when you want to configure db engine args that SqlAlchemy won't parse -# in connection string. -# See https://docs.sqlalchemy.org/en/13/core/engines.html#sqlalchemy.create_engine.params.connect_args -# sql_alchemy_connect_args = - -# The amount of parallelism as a setting to the executor. This defines -# the max number of task instances that should run simultaneously -# on this airflow installation -parallelism = 32 - -# The number of task instances allowed to run concurrently by the scheduler -# in one DAG. Can be overridden by ``concurrency`` on DAG level. -dag_concurrency = 16 - -# Are DAGs paused by default at creation -dags_are_paused_at_creation = True - -# The maximum number of active DAG runs per DAG -max_active_runs_per_dag = 16 - -# Whether to load the DAG examples that ship with Airflow. It's good to -# get started, but you probably want to set this to ``False`` in a production -# environment -load_examples = True - -# Whether to load the default connections that ship with Airflow. It's good to -# get started, but you probably want to set this to ``False`` in a production -# environment -load_default_connections = True - -# Path to the folder containing Airflow plugins -plugins_folder = /usr/local/airflow/plugins - -# Should tasks be executed via forking of the parent process ("False", -# the speedier option) or by spawning a new python process ("True" slow, -# but means plugin changes picked up by tasks straight away) -execute_tasks_new_python_interpreter = False - -# Secret key to save connection passwords in the db -fernet_key = y_xj2l7hU5QKHGn-9o_T9-tIu-pUN1wvXom1ZanIN1w= - -# Whether to disable pickling dags -donot_pickle = True - -# How long before timing out a python file import -dagbag_import_timeout = 30.0 - -# Should a traceback be shown in the UI for dagbag import errors, -# instead of just the exception message -dagbag_import_error_tracebacks = True - -# If tracebacks are shown, how many entries from the traceback should be shown -dagbag_import_error_traceback_depth = 2 - -# How long before timing out a DagFileProcessor, which processes a dag file -dag_file_processor_timeout = 50 - -# The class to use for running task instances in a subprocess. -# Choices include StandardTaskRunner, CgroupTaskRunner or the full import path to the class -# when using a custom task runner. -task_runner = StandardTaskRunner - -# If set, tasks without a ``run_as_user`` argument will be run with this user -# Can be used to de-elevate a sudo user running Airflow when executing tasks -default_impersonation = - -# What security module to use (for example kerberos) -security = - -# Turn unit test mode on (overwrites many configuration options with test -# values at runtime) -unit_test_mode = False - -# Whether to enable pickling for xcom (note that this is insecure and allows for -# RCE exploits). -enable_xcom_pickling = False - -# When a task is killed forcefully, this is the amount of time in seconds that -# it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED -killed_task_cleanup_time = 60 - -# Whether to override params with dag_run.conf. If you pass some key-value pairs -# through ``airflow dags backfill -c`` or -# ``airflow dags trigger -c``, the key-value pairs will override the existing ones in params. -dag_run_conf_overrides_params = True - -# When discovering DAGs, ignore any files that don't contain the strings ``DAG`` and ``airflow``. -dag_discovery_safe_mode = True - -# The number of retries each task is going to have by default. Can be overridden at dag or task level. -default_task_retries = 0 - -# Updating serialized DAG can not be faster than a minimum interval to reduce database write rate. -min_serialized_dag_update_interval = 30 - -# Fetching serialized DAG can not be faster than a minimum interval to reduce database -# read rate. This config controls when your DAGs are updated in the Webserver -min_serialized_dag_fetch_interval = 10 - -# Whether to persist DAG files code in DB. -# If set to True, Webserver reads file contents from DB instead of -# trying to access files in a DAG folder. -# Example: store_dag_code = False -# store_dag_code = - -# Maximum number of Rendered Task Instance Fields (Template Fields) per task to store -# in the Database. -# All the template_fields for each of Task Instance are stored in the Database. -# Keeping this number small may cause an error when you try to view ``Rendered`` tab in -# TaskInstance view for older tasks. -max_num_rendered_ti_fields_per_task = 30 - -# On each dagrun check against defined SLAs -check_slas = True - -# Path to custom XCom class that will be used to store and resolve operators results -# Example: xcom_backend = path.to.CustomXCom -xcom_backend = airflow.models.xcom.BaseXCom - -# By default Airflow plugins are lazily-loaded (only loaded when required). Set it to ``False``, -# if you want to load plugins whenever 'airflow' is invoked via cli or loaded from module. -lazy_load_plugins = True - -# By default Airflow providers are lazily-discovered (discovery and imports happen only when required). -# Set it to False, if you want to discover providers whenever 'airflow' is invoked via cli or -# loaded from module. -lazy_discover_providers = True - -# Number of times the code should be retried in case of DB Operational Errors. -# Not all transactions will be retried as it can cause undesired state. -# Currently it is only used in ``DagFileProcessor.process_file`` to retry ``dagbag.sync_to_db``. -max_db_retries = 3 - -[logging] -# The folder where airflow should store its log files -# This path must be absolute -base_log_folder = /usr/local/airflow/logs - -# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search. -# Set this to True if you want to enable remote logging. -remote_logging = False - -# Users must supply an Airflow connection id that provides access to the storage -# location. -remote_log_conn_id = - -# Path to Google Credential JSON file. If omitted, authorization based on `the Application Default -# Credentials -# `__ will -# be used. -google_key_path = - -# Storage bucket URL for remote logging -# S3 buckets should start with "s3://" -# Cloudwatch log groups should start with "cloudwatch://" -# GCS buckets should start with "gs://" -# WASB buckets should start with "wasb" just to help Airflow select correct handler -# Stackdriver logs should start with "stackdriver://" -remote_base_log_folder = - -# Use server-side encryption for logs stored in S3 -encrypt_s3_logs = False - -# Logging level -logging_level = INFO - -# Logging level for Flask-appbuilder UI -fab_logging_level = WARN - -# Logging class -# Specify the class that will specify the logging configuration -# This class has to be on the python classpath -# Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG -logging_config_class = - -# Flag to enable/disable Colored logs in Console -# Colour the logs when the controlling terminal is a TTY. -colored_console_log = True - -# Log format for when Colored logs is enabled -colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s -colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter - -# Format of Log line -log_format = [%%(asctime)s] {%%(filename)s:%%(lineno)d} %%(levelname)s - %%(message)s -simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s - -# Specify prefix pattern like mentioned below with stream handler TaskHandlerWithCustomFormatter -# Example: task_log_prefix_template = {ti.dag_id}-{ti.task_id}-{execution_date}-{try_number} -task_log_prefix_template = - -# Formatting for how airflow generates file names/paths for each task run. -log_filename_template = {{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log - -# Formatting for how airflow generates file names for log -log_processor_filename_template = {{ filename }}.log - -# full path of dag_processor_manager logfile -dag_processor_manager_log_location = /usr/local/airflow/logs/dag_processor_manager/dag_processor_manager.log - -# Name of handler to read task instance logs. -# Defaults to use ``task`` handler. -task_log_reader = task - -# A comma\-separated list of third-party logger names that will be configured to print messages to -# consoles\. -# Example: extra_loggers = connexion,sqlalchemy -extra_loggers = - -[metrics] - -# StatsD (https://github.com/etsy/statsd) integration settings. -# Enables sending metrics to StatsD. -statsd_on = False -statsd_host = localhost -statsd_port = 8125 -statsd_prefix = airflow - -# If you want to avoid sending all the available metrics to StatsD, -# you can configure an allow list of prefixes (comma separated) to send only the metrics that -# start with the elements of the list (e.g: "scheduler,executor,dagrun") -statsd_allow_list = - -# A function that validate the statsd stat name, apply changes to the stat name if necessary and return -# the transformed stat name. -# -# The function should have the following signature: -# def func_name(stat_name: str) -> str: -stat_name_handler = - -# To enable datadog integration to send airflow metrics. -statsd_datadog_enabled = False - -# List of datadog tags attached to all metrics(e.g: key1:value1,key2:value2) -statsd_datadog_tags = - -# If you want to utilise your own custom Statsd client set the relevant -# module path below. -# Note: The module path must exist on your PYTHONPATH for Airflow to pick it up -# statsd_custom_client_path = - -[secrets] -# Full class name of secrets backend to enable (will precede env vars and metastore in search path) -# Example: backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend -backend = - -# The backend_kwargs param is loaded into a dictionary and passed to __init__ of secrets backend class. -# See documentation for the secrets backend you are using. JSON is expected. -# Example for AWS Systems Manager ParameterStore: -# ``{"connections_prefix": "/airflow/connections", "profile_name": "default"}`` -backend_kwargs = - -[cli] -# In what way should the cli access the API. The LocalClient will use the -# database directly, while the json_client will use the api running on the -# webserver -api_client = airflow.api.client.local_client - -# If you set web_server_url_prefix, do NOT forget to append it here, ex: -# ``endpoint_url = http://localhost:8080/myroot`` -# So api will look like: ``http://localhost:8080/myroot/api/experimental/...`` -endpoint_url = http://localhost:8080 - -[debug] -# Used only with ``DebugExecutor``. If set to ``True`` DAG will fail with first -# failed task. Helpful for debugging purposes. -fail_fast = False - -[api] -# Enables the deprecated experimental API. Please note that these APIs do not have access control. -# The authenticated user has full access. -# -# .. warning:: -# -# This `Experimental REST API `__ is -# deprecated since version 2.0. Please consider using -# `the Stable REST API `__. -# For more information on migration, see -# `UPDATING.md `_ -enable_experimental_api = False - -# How to authenticate users of the API. See -# https://airflow.apache.org/docs/stable/security.html for possible values. -# ("airflow.api.auth.backend.default" allows all requests for historic reasons) -auth_backend = airflow.api.auth.backend.deny_all - -# Used to set the maximum page limit for API requests -maximum_page_limit = 100 - -# Used to set the default page limit when limit is zero. A default limit -# of 100 is set on OpenApi spec. However, this particular default limit -# only work when limit is set equal to zero(0) from API requests. -# If no limit is supplied, the OpenApi spec default is used. -fallback_page_limit = 100 - -# The intended audience for JWT token credentials used for authorization. This value must match on the client and server sides. If empty, audience will not be tested. -# Example: google_oauth2_audience = project-id-random-value.apps.googleusercontent.com -google_oauth2_audience = - -# Path to Google Cloud Service Account key file (JSON). If omitted, authorization based on -# `the Application Default Credentials -# `__ will -# be used. -# Example: google_key_path = /files/service-account-json -google_key_path = - -[lineage] -# what lineage backend to use -backend = - -[atlas] -sasl_enabled = False -host = -port = 21000 -username = -password = - -[operators] -# The default owner assigned to each new operator, unless -# provided explicitly or passed via ``default_args`` -default_owner = airflow -default_cpus = 1 -default_ram = 512 -default_disk = 512 -default_gpus = 0 - -# Is allowed to pass additional/unused arguments (args, kwargs) to the BaseOperator operator. -# If set to False, an exception will be thrown, otherwise only the console message will be displayed. -allow_illegal_arguments = False - -[hive] -# Default mapreduce queue for HiveOperator tasks -default_hive_mapred_queue = - -# Template for mapred_job_name in HiveOperator, supports the following named parameters -# hostname, dag_id, task_id, execution_date -# mapred_job_name_template = - -[webserver] -# The base url of your website as airflow cannot guess what domain or -# cname you are using. This is used in automated emails that -# airflow sends to point links to the right web server -base_url = http://localhost:8080 - -# Default timezone to display all dates in the UI, can be UTC, system, or -# any IANA timezone string (e.g. Europe/Amsterdam). If left empty the -# default value of core/default_timezone will be used -# Example: default_ui_timezone = America/New_York -default_ui_timezone = UTC - -# The ip specified when starting the web server -web_server_host = 0.0.0.0 - -# The port on which to run the web server -web_server_port = 8080 - -# Paths to the SSL certificate and key for the web server. When both are -# provided SSL will be enabled. This does not change the web server port. -web_server_ssl_cert = - -# Paths to the SSL certificate and key for the web server. When both are -# provided SSL will be enabled. This does not change the web server port. -web_server_ssl_key = - -# Number of seconds the webserver waits before killing gunicorn master that doesn't respond -web_server_master_timeout = 120 - -# Number of seconds the gunicorn webserver waits before timing out on a worker -web_server_worker_timeout = 120 - -# Number of workers to refresh at a time. When set to 0, worker refresh is -# disabled. When nonzero, airflow periodically refreshes webserver workers by -# bringing up new ones and killing old ones. -worker_refresh_batch_size = 1 - -# Number of seconds to wait before refreshing a batch of workers. -worker_refresh_interval = 30 - -# If set to True, Airflow will track files in plugins_folder directory. When it detects changes, -# then reload the gunicorn. -reload_on_plugin_change = False - -# Secret key used to run your flask app -# It should be as random as possible -secret_key = 5b5GOkm5PddhiGyFsoEZFw== - -# Number of workers to run the Gunicorn web server -workers = 4 - -# The worker class gunicorn should use. Choices include -# sync (default), eventlet, gevent -worker_class = sync - -# Log files for the gunicorn webserver. '-' means log to stderr. -access_logfile = - - -# Log files for the gunicorn webserver. '-' means log to stderr. -error_logfile = - - -# Access log format for gunicorn webserver. -# default format is %%(h)s %%(l)s %%(u)s %%(t)s "%%(r)s" %%(s)s %%(b)s "%%(f)s" "%%(a)s" -# documentation - https://docs.gunicorn.org/en/stable/settings.html#access-log-format -access_logformat = - -# Expose the configuration file in the web server -expose_config = False - -# Expose hostname in the web server -expose_hostname = True - -# Expose stacktrace in the web server -expose_stacktrace = True - -# Default DAG view. Valid values are: ``tree``, ``graph``, ``duration``, ``gantt``, ``landing_times`` -dag_default_view = tree - -# Default DAG orientation. Valid values are: -# ``LR`` (Left->Right), ``TB`` (Top->Bottom), ``RL`` (Right->Left), ``BT`` (Bottom->Top) -dag_orientation = LR - -# Puts the webserver in demonstration mode; blurs the names of Operators for -# privacy. -demo_mode = False - -# The amount of time (in secs) webserver will wait for initial handshake -# while fetching logs from other worker machine -log_fetch_timeout_sec = 5 - -# Time interval (in secs) to wait before next log fetching. -log_fetch_delay_sec = 2 - -# Distance away from page bottom to enable auto tailing. -log_auto_tailing_offset = 30 - -# Animation speed for auto tailing log display. -log_animation_speed = 1000 - -# By default, the webserver shows paused DAGs. Flip this to hide paused -# DAGs by default -hide_paused_dags_by_default = False - -# Consistent page size across all listing views in the UI -page_size = 100 - -# Define the color of navigation bar -navbar_color = #fff - -# Default dagrun to show in UI -default_dag_run_display_number = 25 - -# Enable werkzeug ``ProxyFix`` middleware for reverse proxy -enable_proxy_fix = False - -# Number of values to trust for ``X-Forwarded-For``. -# More info: https://werkzeug.palletsprojects.com/en/0.16.x/middleware/proxy_fix/ -proxy_fix_x_for = 1 - -# Number of values to trust for ``X-Forwarded-Proto`` -proxy_fix_x_proto = 1 - -# Number of values to trust for ``X-Forwarded-Host`` -proxy_fix_x_host = 1 - -# Number of values to trust for ``X-Forwarded-Port`` -proxy_fix_x_port = 1 - -# Number of values to trust for ``X-Forwarded-Prefix`` -proxy_fix_x_prefix = 1 - -# Set secure flag on session cookie -cookie_secure = False - -# Set samesite policy on session cookie -cookie_samesite = Lax - -# Default setting for wrap toggle on DAG code and TI log views. -default_wrap = False - -# Allow the UI to be rendered in a frame -x_frame_enabled = True - -# Send anonymous user activity to your analytics tool -# choose from google_analytics, segment, or metarouter -# analytics_tool = - -# Unique ID of your account in the analytics tool -# analytics_id = - -# 'Recent Tasks' stats will show for old DagRuns if set -show_recent_stats_for_completed_runs = True - -# Update FAB permissions and sync security manager roles -# on webserver startup -update_fab_perms = True - -# The UI cookie lifetime in minutes. User will be logged out from UI after -# ``session_lifetime_minutes`` of non-activity -session_lifetime_minutes = 43200 - -[email] - -# Configuration email backend and whether to -# send email alerts on retry or failure -# Email backend to use -email_backend = airflow.utils.email.send_email_smtp - -# Whether email alerts should be sent when a task is retried -default_email_on_retry = True - -# Whether email alerts should be sent when a task failed -default_email_on_failure = True - -[smtp] - -# If you want airflow to send emails on retries, failure, and you want to use -# the airflow.utils.email.send_email_smtp function, you have to configure an -# smtp server here -smtp_host = localhost -smtp_starttls = True -smtp_ssl = False -# Example: smtp_user = airflow -# smtp_user = -# Example: smtp_password = airflow -# smtp_password = -smtp_port = 25 -smtp_mail_from = airflow@example.com -smtp_timeout = 30 -smtp_retry_limit = 5 - -[sentry] - -# Sentry (https://docs.sentry.io) integration. Here you can supply -# additional configuration options based on the Python platform. See: -# https://docs.sentry.io/error-reporting/configuration/?platform=python. -# Unsupported options: ``integrations``, ``in_app_include``, ``in_app_exclude``, -# ``ignore_errors``, ``before_breadcrumb``, ``before_send``, ``transport``. -# Enable error reporting to Sentry -sentry_on = false -sentry_dsn = - -[celery_kubernetes_executor] - -# This section only applies if you are using the ``CeleryKubernetesExecutor`` in -# ``[core]`` section above -# Define when to send a task to ``KubernetesExecutor`` when using ``CeleryKubernetesExecutor``. -# When the queue of a task is ``kubernetes_queue``, the task is executed via ``KubernetesExecutor``, -# otherwise via ``CeleryExecutor`` -kubernetes_queue = kubernetes - -[celery] - -# This section only applies if you are using the CeleryExecutor in -# ``[core]`` section above -# The app name that will be used by celery -celery_app_name = airflow.executors.celery_executor - -# The concurrency that will be used when starting workers with the -# ``airflow celery worker`` command. This defines the number of task instances that -# a worker will take, so size up your workers based on the resources on -# your worker box and the nature of your tasks -worker_concurrency = 8 - -# The maximum and minimum concurrency that will be used when starting workers with the -# ``airflow celery worker`` command (always keep minimum processes, but grow -# to maximum if necessary). Note the value should be max_concurrency,min_concurrency -# Pick these numbers based on resources on worker box and the nature of the task. -# If autoscale option is available, worker_concurrency will be ignored. -# http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale -# Example: worker_autoscale = 16,12 -# worker_autoscale = - -# Used to increase the number of tasks that a worker prefetches which can improve performance. -# The number of processes multiplied by worker_prefetch_multiplier is the number of tasks -# that are prefetched by a worker. A value greater than 1 can result in tasks being unnecessarily -# blocked if there are multiple workers and one worker prefetches tasks that sit behind long -# running tasks while another worker has unutilized processes that are unable to process the already -# claimed blocked tasks. -# https://docs.celeryproject.org/en/stable/userguide/optimizing.html#prefetch-limits -# Example: worker_prefetch_multiplier = 1 -# worker_prefetch_multiplier = - -# When you start an airflow worker, airflow starts a tiny web server -# subprocess to serve the workers local log files to the airflow main -# web server, who then builds pages and sends them to users. This defines -# the port on which the logs are served. It needs to be unused, and open -# visible from the main web server to connect into the workers. -worker_log_server_port = 8793 - -# Umask that will be used when starting workers with the ``airflow celery worker`` -# in daemon mode. This control the file-creation mode mask which determines the initial -# value of file permission bits for newly created files. -worker_umask = 0o077 - -# The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally -# a sqlalchemy database. Refer to the Celery documentation for more information. -broker_url = redis://redis:6379/0 - -# The Celery result_backend. When a job finishes, it needs to update the -# metadata of the job. Therefore it will post a message on a message bus, -# or insert it into a database (depending of the backend) -# This status is used by the scheduler to update the state of the task -# The use of a database is highly recommended -# http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings -result_backend = db+postgresql://postgres:airflow@postgres/airflow - -# Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start -# it ``airflow celery flower``. This defines the IP that Celery Flower runs on -flower_host = 0.0.0.0 - -# The root URL for Flower -# Example: flower_url_prefix = /flower -flower_url_prefix = - -# This defines the port that Celery Flower runs on -flower_port = 5555 - -# Securing Flower with Basic Authentication -# Accepts user:password pairs separated by a comma -# Example: flower_basic_auth = user1:password1,user2:password2 -flower_basic_auth = - -# Default queue that tasks get assigned to and that worker listen on. -default_queue = default - -# How many processes CeleryExecutor uses to sync task state. -# 0 means to use max(1, number of cores - 1) processes. -sync_parallelism = 0 - -# Import path for celery configuration options -celery_config_options = airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG -ssl_active = False -ssl_key = -ssl_cert = -ssl_cacert = - -# Celery Pool implementation. -# Choices include: ``prefork`` (default), ``eventlet``, ``gevent`` or ``solo``. -# See: -# https://docs.celeryproject.org/en/latest/userguide/workers.html#concurrency -# https://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html -pool = prefork - -# The number of seconds to wait before timing out ``send_task_to_executor`` or -# ``fetch_celery_task_state`` operations. -operation_timeout = 1.0 - -# Celery task will report its status as 'started' when the task is executed by a worker. -# This is used in Airflow to keep track of the running tasks and if a Scheduler is restarted -# or run in HA mode, it can adopt the orphan tasks launched by previous SchedulerJob. -task_track_started = True - -# Time in seconds after which Adopted tasks are cleared by CeleryExecutor. This is helpful to clear -# stalled tasks. -task_adoption_timeout = 600 - -# The Maximum number of retries for publishing task messages to the broker when failing -# due to ``AirflowTaskTimeout`` error before giving up and marking Task as failed. -task_publish_max_retries = 3 - -# Worker initialisation check to validate Metadata Database connection -worker_precheck = False - -[celery_broker_transport_options] - -# This section is for specifying options which can be passed to the -# underlying celery broker transport. See: -# http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_transport_options -# The visibility timeout defines the number of seconds to wait for the worker -# to acknowledge the task before the message is redelivered to another worker. -# Make sure to increase the visibility timeout to match the time of the longest -# ETA you're planning to use. -# visibility_timeout is only supported for Redis and SQS celery brokers. -# See: -# http://docs.celeryproject.org/en/master/userguide/configuration.html#std:setting-broker_transport_options -# Example: visibility_timeout = 21600 -# visibility_timeout = - -[dask] - -# This section only applies if you are using the DaskExecutor in -# [core] section above -# The IP address and port of the Dask cluster's scheduler. -cluster_address = 127.0.0.1:8786 - -# TLS/ SSL settings to access a secured Dask scheduler. -tls_ca = -tls_cert = -tls_key = - -[scheduler] -# Task instances listen for external kill signal (when you clear tasks -# from the CLI or the UI), this defines the frequency at which they should -# listen (in seconds). -job_heartbeat_sec = 5 - -# How often (in seconds) to check and tidy up 'running' TaskInstancess -# that no longer have a matching DagRun -clean_tis_without_dagrun_interval = 15.0 - -# The scheduler constantly tries to trigger new tasks (look at the -# scheduler section in the docs for more information). This defines -# how often the scheduler should run (in seconds). -scheduler_heartbeat_sec = 5 - -# The number of times to try to schedule each DAG file -# -1 indicates unlimited number -num_runs = -1 - -# The number of seconds to wait between consecutive DAG file processing -processor_poll_interval = 1 - -# after how much time (seconds) a new DAGs should be picked up from the filesystem -min_file_process_interval = 0 - -# How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes. -dag_dir_list_interval = 300 - -# How often should stats be printed to the logs. Setting to 0 will disable printing stats -print_stats_interval = 30 - -# How often (in seconds) should pool usage stats be sent to statsd (if statsd_on is enabled) -pool_metrics_interval = 5.0 - -# If the last scheduler heartbeat happened more than scheduler_health_check_threshold -# ago (in seconds), scheduler is considered unhealthy. -# This is used by the health check in the "/health" endpoint -scheduler_health_check_threshold = 30 - -# How often (in seconds) should the scheduler check for orphaned tasks and SchedulerJobs -orphaned_tasks_check_interval = 300.0 -child_process_log_directory = /usr/local/airflow/logs/scheduler - -# Local task jobs periodically heartbeat to the DB. If the job has -# not heartbeat in this many seconds, the scheduler will mark the -# associated task instance as failed and will re-schedule the task. -scheduler_zombie_task_threshold = 300 - -# Turn off scheduler catchup by setting this to ``False``. -# Default behavior is unchanged and -# Command Line Backfills still work, but the scheduler -# will not do scheduler catchup if this is ``False``, -# however it can be set on a per DAG basis in the -# DAG definition (catchup) -catchup_by_default = True - -# This changes the batch size of queries in the scheduling main loop. -# If this is too high, SQL query performance may be impacted by one -# or more of the following: -# - reversion to full table scan -# - complexity of query predicate -# - excessive locking -# Additionally, you may hit the maximum allowable query length for your db. -# Set this to 0 for no limit (not advised) -max_tis_per_query = 512 - -# Should the scheduler issue ``SELECT ... FOR UPDATE`` in relevant queries. -# If this is set to False then you should not run more than a single -# scheduler at once -use_row_level_locking = True - -# Max number of DAGs to create DagRuns for per scheduler loop -# -# Default: 10 -# max_dagruns_to_create_per_loop = - -# How many DagRuns should a scheduler examine (and lock) when scheduling -# and queuing tasks. -# -# Default: 20 -# max_dagruns_per_loop_to_schedule = - -# Should the Task supervisor process perform a "mini scheduler" to attempt to schedule more tasks of the -# same DAG. Leaving this on will mean tasks in the same DAG execute quicker, but might starve out other -# dags in some circumstances -# -# Default: True -# schedule_after_task_execution = - -# The scheduler can run multiple processes in parallel to parse dags. -# This defines how many processes will run. -parsing_processes = 2 - -# Turn off scheduler use of cron intervals by setting this to False. -# DAGs submitted manually in the web UI or with trigger_dag will still run. -use_job_schedule = True - -# Allow externally triggered DagRuns for Execution Dates in the future -# Only has effect if schedule_interval is set to None in DAG -allow_trigger_in_future = False - -[kerberos] -ccache = /tmp/airflow_krb5_ccache - -# gets augmented with fqdn -principal = airflow -reinit_frequency = 3600 -kinit_path = kinit -keytab = airflow.keytab - -[github_enterprise] -api_rev = v3 - -[admin] -# UI to hide sensitive variable fields when set to True -hide_sensitive_variable_fields = True - -# A comma-separated list of sensitive keywords to look for in variables names. -sensitive_variable_fields = - -[elasticsearch] -# Elasticsearch host -host = - -# Format of the log_id, which is used to query for a given tasks logs -log_id_template = {dag_id}-{task_id}-{execution_date}-{try_number} - -# Used to mark the end of a log stream for a task -end_of_log_mark = end_of_log - -# Qualified URL for an elasticsearch frontend (like Kibana) with a template argument for log_id -# Code will construct log_id using the log_id template from the argument above. -# NOTE: The code will prefix the https:// automatically, don't include that here. -frontend = - -# Write the task logs to the stdout of the worker, rather than the default files -write_stdout = False - -# Instead of the default log formatter, write the log lines as JSON -json_format = False - -# Log fields to also attach to the json output, if enabled -json_fields = asctime, filename, lineno, levelname, message - -[elasticsearch_configs] -use_ssl = False -verify_certs = True - -[kubernetes] -# Path to the YAML pod file. If set, all other kubernetes-related fields are ignored. -pod_template_file = - -# The repository of the Kubernetes Image for the Worker to Run -worker_container_repository = - -# The tag of the Kubernetes Image for the Worker to Run -worker_container_tag = - -# The Kubernetes namespace where airflow workers should be created. Defaults to ``default`` -namespace = default - -# If True, all worker pods will be deleted upon termination -delete_worker_pods = True - -# If False (and delete_worker_pods is True), -# failed worker pods will not be deleted so users can investigate them. -delete_worker_pods_on_failure = False - -# Number of Kubernetes Worker Pod creation calls per scheduler loop. -# Note that the current default of "1" will only launch a single pod -# per-heartbeat. It is HIGHLY recommended that users increase this -# number to match the tolerance of their kubernetes cluster for -# better performance. -worker_pods_creation_batch_size = 1 - -# Allows users to launch pods in multiple namespaces. -# Will require creating a cluster-role for the scheduler -multi_namespace_mode = False - -# Use the service account kubernetes gives to pods to connect to kubernetes cluster. -# It's intended for clients that expect to be running inside a pod running on kubernetes. -# It will raise an exception if called from a process not running in a kubernetes environment. -in_cluster = True - -# When running with in_cluster=False change the default cluster_context or config_file -# options to Kubernetes client. Leave blank these to use default behaviour like ``kubectl`` has. -# cluster_context = - -# Path to the kubernetes configfile to be used when ``in_cluster`` is set to False -# config_file = - -# Keyword parameters to pass while calling a kubernetes client core_v1_api methods -# from Kubernetes Executor provided as a single line formatted JSON dictionary string. -# List of supported params are similar for all core_v1_apis, hence a single config -# variable for all apis. See: -# https://raw.githubusercontent.com/kubernetes-client/python/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/api/core_v1_api.py -kube_client_request_args = - -# Optional keyword arguments to pass to the ``delete_namespaced_pod`` kubernetes client -# ``core_v1_api`` method when using the Kubernetes Executor. -# This should be an object and can contain any of the options listed in the ``v1DeleteOptions`` -# class defined here: -# https://github.com/kubernetes-client/python/blob/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/models/v1_delete_options.py#L19 -# Example: delete_option_kwargs = {"grace_period_seconds": 10} -delete_option_kwargs = - -# Enables TCP keepalive mechanism. This prevents Kubernetes API requests to hang indefinitely -# when idle connection is time-outed on services like cloud load balancers or firewalls. -enable_tcp_keepalive = False - -# When the `enable_tcp_keepalive` option is enabled, TCP probes a connection that has -# been idle for `tcp_keep_idle` seconds. -tcp_keep_idle = 120 - -# When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond -# to a keepalive probe, TCP retransmits the probe after `tcp_keep_intvl` seconds. -tcp_keep_intvl = 30 - -# When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond -# to a keepalive probe, TCP retransmits the probe `tcp_keep_cnt number` of times before -# a connection is considered to be broken. -tcp_keep_cnt = 6 - -[smart_sensor] -# When `use_smart_sensor` is True, Airflow redirects multiple qualified sensor tasks to -# smart sensor task. -use_smart_sensor = False - -# `shard_code_upper_limit` is the upper limit of `shard_code` value. The `shard_code` is generated -# by `hashcode % shard_code_upper_limit`. -shard_code_upper_limit = 10000 - -# The number of running smart sensor processes for each service. -shards = 5 - -# comma separated sensor classes support in smart_sensor. -sensors_enabled = NamedHivePartitionSensor +[core] +# The folder where your airflow pipelines live, most likely a +# subfolder in a code repository. This path must be absolute. +dags_folder = /usr/local/airflow/dags + +# Hostname by providing a path to a callable, which will resolve the hostname. +# The format is "package.function". +# +# For example, default value "socket.getfqdn" means that result from getfqdn() of "socket" +# package will be used as hostname. +# +# No argument should be required in the function specified. +# If using IP address as hostname is preferred, use value ``airflow.utils.net.get_host_ip_address`` +hostname_callable = socket.getfqdn + +# Default timezone in case supplied date times are naive +# can be utc (default), system, or any IANA timezone string (e.g. Europe/Amsterdam) +default_timezone = utc + +# The executor class that airflow should use. Choices include +# ``SequentialExecutor``, ``LocalExecutor``, ``CeleryExecutor``, ``DaskExecutor``, +# ``KubernetesExecutor``, ``CeleryKubernetesExecutor`` or the +# full import path to the class when using a custom executor. +executor = SequentialExecutor + +# The SqlAlchemy connection string to the metadata database. +# SqlAlchemy supports many different database engine, more information +# their website +sql_alchemy_conn = sqlite:////usr/local/airflow/airflow.db + +# The encoding for the databases +sql_engine_encoding = utf-8 + +# Collation for ``dag_id``, ``task_id``, ``key`` columns in case they have different encoding. +# This is particularly useful in case of mysql with utf8mb4 encoding because +# primary keys for XCom table has too big size and ``sql_engine_collation_for_ids`` should +# be set to ``utf8mb3_general_ci``. +# sql_engine_collation_for_ids = + +# If SqlAlchemy should pool database connections. +sql_alchemy_pool_enabled = True + +# The SqlAlchemy pool size is the maximum number of database connections +# in the pool. 0 indicates no limit. +sql_alchemy_pool_size = 5 + +# The maximum overflow size of the pool. +# When the number of checked-out connections reaches the size set in pool_size, +# additional connections will be returned up to this limit. +# When those additional connections are returned to the pool, they are disconnected and discarded. +# It follows then that the total number of simultaneous connections the pool will allow +# is pool_size + max_overflow, +# and the total number of "sleeping" connections the pool will allow is pool_size. +# max_overflow can be set to ``-1`` to indicate no overflow limit; +# no limit will be placed on the total number of concurrent connections. Defaults to ``10``. +sql_alchemy_max_overflow = 10 + +# The SqlAlchemy pool recycle is the number of seconds a connection +# can be idle in the pool before it is invalidated. This config does +# not apply to sqlite. If the number of DB connections is ever exceeded, +# a lower config value will allow the system to recover faster. +sql_alchemy_pool_recycle = 1800 + +# Check connection at the start of each connection pool checkout. +# Typically, this is a simple statement like "SELECT 1". +# More information here: +# https://docs.sqlalchemy.org/en/13/core/pooling.html#disconnect-handling-pessimistic +sql_alchemy_pool_pre_ping = True + +# The schema to use for the metadata database. +# SqlAlchemy supports databases with the concept of multiple schemas. +sql_alchemy_schema = + +# Import path for connect args in SqlAlchemy. Defaults to an empty dict. +# This is useful when you want to configure db engine args that SqlAlchemy won't parse +# in connection string. +# See https://docs.sqlalchemy.org/en/13/core/engines.html#sqlalchemy.create_engine.params.connect_args +# sql_alchemy_connect_args = + +# The amount of parallelism as a setting to the executor. This defines +# the max number of task instances that should run simultaneously +# on this airflow installation +parallelism = 32 + +# The number of task instances allowed to run concurrently by the scheduler +# in one DAG. Can be overridden by ``concurrency`` on DAG level. +dag_concurrency = 16 + +# Are DAGs paused by default at creation +dags_are_paused_at_creation = True + +# The maximum number of active DAG runs per DAG +max_active_runs_per_dag = 16 + +# Whether to load the DAG examples that ship with Airflow. It's good to +# get started, but you probably want to set this to ``False`` in a production +# environment +load_examples = True + +# Whether to load the default connections that ship with Airflow. It's good to +# get started, but you probably want to set this to ``False`` in a production +# environment +load_default_connections = True + +# Path to the folder containing Airflow plugins +plugins_folder = /usr/local/airflow/plugins + +# Should tasks be executed via forking of the parent process ("False", +# the speedier option) or by spawning a new python process ("True" slow, +# but means plugin changes picked up by tasks straight away) +execute_tasks_new_python_interpreter = False + +# Secret key to save connection passwords in the db +fernet_key = y_xj2l7hU5QKHGn-9o_T9-tIu-pUN1wvXom1ZanIN1w= + +# Whether to disable pickling dags +donot_pickle = True + +# How long before timing out a python file import +dagbag_import_timeout = 30.0 + +# Should a traceback be shown in the UI for dagbag import errors, +# instead of just the exception message +dagbag_import_error_tracebacks = True + +# If tracebacks are shown, how many entries from the traceback should be shown +dagbag_import_error_traceback_depth = 2 + +# How long before timing out a DagFileProcessor, which processes a dag file +dag_file_processor_timeout = 50 + +# The class to use for running task instances in a subprocess. +# Choices include StandardTaskRunner, CgroupTaskRunner or the full import path to the class +# when using a custom task runner. +task_runner = StandardTaskRunner + +# If set, tasks without a ``run_as_user`` argument will be run with this user +# Can be used to de-elevate a sudo user running Airflow when executing tasks +default_impersonation = + +# What security module to use (for example kerberos) +security = + +# Turn unit test mode on (overwrites many configuration options with test +# values at runtime) +unit_test_mode = False + +# Whether to enable pickling for xcom (note that this is insecure and allows for +# RCE exploits). +enable_xcom_pickling = False + +# When a task is killed forcefully, this is the amount of time in seconds that +# it has to cleanup after it is sent a SIGTERM, before it is SIGKILLED +killed_task_cleanup_time = 60 + +# Whether to override params with dag_run.conf. If you pass some key-value pairs +# through ``airflow dags backfill -c`` or +# ``airflow dags trigger -c``, the key-value pairs will override the existing ones in params. +dag_run_conf_overrides_params = True + +# When discovering DAGs, ignore any files that don't contain the strings ``DAG`` and ``airflow``. +dag_discovery_safe_mode = True + +# The number of retries each task is going to have by default. Can be overridden at dag or task level. +default_task_retries = 0 + +# Updating serialized DAG can not be faster than a minimum interval to reduce database write rate. +min_serialized_dag_update_interval = 30 + +# Fetching serialized DAG can not be faster than a minimum interval to reduce database +# read rate. This config controls when your DAGs are updated in the Webserver +min_serialized_dag_fetch_interval = 10 + +# Whether to persist DAG files code in DB. +# If set to True, Webserver reads file contents from DB instead of +# trying to access files in a DAG folder. +# Example: store_dag_code = False +# store_dag_code = + +# Maximum number of Rendered Task Instance Fields (Template Fields) per task to store +# in the Database. +# All the template_fields for each of Task Instance are stored in the Database. +# Keeping this number small may cause an error when you try to view ``Rendered`` tab in +# TaskInstance view for older tasks. +max_num_rendered_ti_fields_per_task = 30 + +# On each dagrun check against defined SLAs +check_slas = True + +# Path to custom XCom class that will be used to store and resolve operators results +# Example: xcom_backend = path.to.CustomXCom +xcom_backend = airflow.models.xcom.BaseXCom + +# By default Airflow plugins are lazily-loaded (only loaded when required). Set it to ``False``, +# if you want to load plugins whenever 'airflow' is invoked via cli or loaded from module. +lazy_load_plugins = True + +# By default Airflow providers are lazily-discovered (discovery and imports happen only when required). +# Set it to False, if you want to discover providers whenever 'airflow' is invoked via cli or +# loaded from module. +lazy_discover_providers = True + +# Number of times the code should be retried in case of DB Operational Errors. +# Not all transactions will be retried as it can cause undesired state. +# Currently it is only used in ``DagFileProcessor.process_file`` to retry ``dagbag.sync_to_db``. +max_db_retries = 3 + +[logging] +# The folder where airflow should store its log files +# This path must be absolute +base_log_folder = /usr/local/airflow/logs + +# Airflow can store logs remotely in AWS S3, Google Cloud Storage or Elastic Search. +# Set this to True if you want to enable remote logging. +remote_logging = False + +# Users must supply an Airflow connection id that provides access to the storage +# location. +remote_log_conn_id = + +# Path to Google Credential JSON file. If omitted, authorization based on `the Application Default +# Credentials +# `__ will +# be used. +google_key_path = + +# Storage bucket URL for remote logging +# S3 buckets should start with "s3://" +# Cloudwatch log groups should start with "cloudwatch://" +# GCS buckets should start with "gs://" +# WASB buckets should start with "wasb" just to help Airflow select correct handler +# Stackdriver logs should start with "stackdriver://" +remote_base_log_folder = + +# Use server-side encryption for logs stored in S3 +encrypt_s3_logs = False + +# Logging level +logging_level = INFO + +# Logging level for Flask-appbuilder UI +fab_logging_level = WARN + +# Logging class +# Specify the class that will specify the logging configuration +# This class has to be on the python classpath +# Example: logging_config_class = my.path.default_local_settings.LOGGING_CONFIG +logging_config_class = + +# Flag to enable/disable Colored logs in Console +# Colour the logs when the controlling terminal is a TTY. +colored_console_log = True + +# Log format for when Colored logs is enabled +colored_log_format = [%%(blue)s%%(asctime)s%%(reset)s] {%%(blue)s%%(filename)s:%%(reset)s%%(lineno)d} %%(log_color)s%%(levelname)s%%(reset)s - %%(log_color)s%%(message)s%%(reset)s +colored_formatter_class = airflow.utils.log.colored_log.CustomTTYColoredFormatter + +# Format of Log line +log_format = [%%(asctime)s] {%%(filename)s:%%(lineno)d} %%(levelname)s - %%(message)s +simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s + +# Specify prefix pattern like mentioned below with stream handler TaskHandlerWithCustomFormatter +# Example: task_log_prefix_template = {ti.dag_id}-{ti.task_id}-{execution_date}-{try_number} +task_log_prefix_template = + +# Formatting for how airflow generates file names/paths for each task run. +log_filename_template = {{ ti.dag_id }}/{{ ti.task_id }}/{{ ts }}/{{ try_number }}.log + +# Formatting for how airflow generates file names for log +log_processor_filename_template = {{ filename }}.log + +# full path of dag_processor_manager logfile +dag_processor_manager_log_location = /usr/local/airflow/logs/dag_processor_manager/dag_processor_manager.log + +# Name of handler to read task instance logs. +# Defaults to use ``task`` handler. +task_log_reader = task + +# A comma\-separated list of third-party logger names that will be configured to print messages to +# consoles\. +# Example: extra_loggers = connexion,sqlalchemy +extra_loggers = + +[metrics] + +# StatsD (https://github.com/etsy/statsd) integration settings. +# Enables sending metrics to StatsD. +statsd_on = False +statsd_host = localhost +statsd_port = 8125 +statsd_prefix = airflow + +# If you want to avoid sending all the available metrics to StatsD, +# you can configure an allow list of prefixes (comma separated) to send only the metrics that +# start with the elements of the list (e.g: "scheduler,executor,dagrun") +statsd_allow_list = + +# A function that validate the statsd stat name, apply changes to the stat name if necessary and return +# the transformed stat name. +# +# The function should have the following signature: +# def func_name(stat_name: str) -> str: +stat_name_handler = + +# To enable datadog integration to send airflow metrics. +statsd_datadog_enabled = False + +# List of datadog tags attached to all metrics(e.g: key1:value1,key2:value2) +statsd_datadog_tags = + +# If you want to utilise your own custom Statsd client set the relevant +# module path below. +# Note: The module path must exist on your PYTHONPATH for Airflow to pick it up +# statsd_custom_client_path = + +[secrets] +# Full class name of secrets backend to enable (will precede env vars and metastore in search path) +# Example: backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend +backend = + +# The backend_kwargs param is loaded into a dictionary and passed to __init__ of secrets backend class. +# See documentation for the secrets backend you are using. JSON is expected. +# Example for AWS Systems Manager ParameterStore: +# ``{"connections_prefix": "/airflow/connections", "profile_name": "default"}`` +backend_kwargs = + +[cli] +# In what way should the cli access the API. The LocalClient will use the +# database directly, while the json_client will use the api running on the +# webserver +api_client = airflow.api.client.local_client + +# If you set web_server_url_prefix, do NOT forget to append it here, ex: +# ``endpoint_url = http://localhost:8080/myroot`` +# So api will look like: ``http://localhost:8080/myroot/api/experimental/...`` +endpoint_url = http://localhost:8080 + +[debug] +# Used only with ``DebugExecutor``. If set to ``True`` DAG will fail with first +# failed task. Helpful for debugging purposes. +fail_fast = False + +[api] +# Enables the deprecated experimental API. Please note that these APIs do not have access control. +# The authenticated user has full access. +# +# .. warning:: +# +# This `Experimental REST API `__ is +# deprecated since version 2.0. Please consider using +# `the Stable REST API `__. +# For more information on migration, see +# `UPDATING.md `_ +enable_experimental_api = False + +# How to authenticate users of the API. See +# https://airflow.apache.org/docs/stable/security.html for possible values. +# ("airflow.api.auth.backend.default" allows all requests for historic reasons) +auth_backend = airflow.api.auth.backend.deny_all + +# Used to set the maximum page limit for API requests +maximum_page_limit = 100 + +# Used to set the default page limit when limit is zero. A default limit +# of 100 is set on OpenApi spec. However, this particular default limit +# only work when limit is set equal to zero(0) from API requests. +# If no limit is supplied, the OpenApi spec default is used. +fallback_page_limit = 100 + +# The intended audience for JWT token credentials used for authorization. This value must match on the client and server sides. If empty, audience will not be tested. +# Example: google_oauth2_audience = project-id-random-value.apps.googleusercontent.com +google_oauth2_audience = + +# Path to Google Cloud Service Account key file (JSON). If omitted, authorization based on +# `the Application Default Credentials +# `__ will +# be used. +# Example: google_key_path = /files/service-account-json +google_key_path = + +[lineage] +# what lineage backend to use +backend = + +[atlas] +sasl_enabled = False +host = +port = 21000 +username = +password = + +[operators] +# The default owner assigned to each new operator, unless +# provided explicitly or passed via ``default_args`` +default_owner = airflow +default_cpus = 1 +default_ram = 512 +default_disk = 512 +default_gpus = 0 + +# Is allowed to pass additional/unused arguments (args, kwargs) to the BaseOperator operator. +# If set to False, an exception will be thrown, otherwise only the console message will be displayed. +allow_illegal_arguments = False + +[hive] +# Default mapreduce queue for HiveOperator tasks +default_hive_mapred_queue = + +# Template for mapred_job_name in HiveOperator, supports the following named parameters +# hostname, dag_id, task_id, execution_date +# mapred_job_name_template = + +[webserver] +# The base url of your website as airflow cannot guess what domain or +# cname you are using. This is used in automated emails that +# airflow sends to point links to the right web server +base_url = http://localhost:8080 + +# Default timezone to display all dates in the UI, can be UTC, system, or +# any IANA timezone string (e.g. Europe/Amsterdam). If left empty the +# default value of core/default_timezone will be used +# Example: default_ui_timezone = America/New_York +default_ui_timezone = UTC + +# The ip specified when starting the web server +web_server_host = 0.0.0.0 + +# The port on which to run the web server +web_server_port = 8080 + +# Paths to the SSL certificate and key for the web server. When both are +# provided SSL will be enabled. This does not change the web server port. +web_server_ssl_cert = + +# Paths to the SSL certificate and key for the web server. When both are +# provided SSL will be enabled. This does not change the web server port. +web_server_ssl_key = + +# Number of seconds the webserver waits before killing gunicorn master that doesn't respond +web_server_master_timeout = 120 + +# Number of seconds the gunicorn webserver waits before timing out on a worker +web_server_worker_timeout = 120 + +# Number of workers to refresh at a time. When set to 0, worker refresh is +# disabled. When nonzero, airflow periodically refreshes webserver workers by +# bringing up new ones and killing old ones. +worker_refresh_batch_size = 1 + +# Number of seconds to wait before refreshing a batch of workers. +worker_refresh_interval = 30 + +# If set to True, Airflow will track files in plugins_folder directory. When it detects changes, +# then reload the gunicorn. +reload_on_plugin_change = False + +# Secret key used to run your flask app +# It should be as random as possible +secret_key = 5b5GOkm5PddhiGyFsoEZFw== + +# Number of workers to run the Gunicorn web server +workers = 4 + +# The worker class gunicorn should use. Choices include +# sync (default), eventlet, gevent +worker_class = sync + +# Log files for the gunicorn webserver. '-' means log to stderr. +access_logfile = - + +# Log files for the gunicorn webserver. '-' means log to stderr. +error_logfile = - + +# Access log format for gunicorn webserver. +# default format is %%(h)s %%(l)s %%(u)s %%(t)s "%%(r)s" %%(s)s %%(b)s "%%(f)s" "%%(a)s" +# documentation - https://docs.gunicorn.org/en/stable/settings.html#access-log-format +access_logformat = + +# Expose the configuration file in the web server +expose_config = False + +# Expose hostname in the web server +expose_hostname = True + +# Expose stacktrace in the web server +expose_stacktrace = True + +# Default DAG view. Valid values are: ``tree``, ``graph``, ``duration``, ``gantt``, ``landing_times`` +dag_default_view = tree + +# Default DAG orientation. Valid values are: +# ``LR`` (Left->Right), ``TB`` (Top->Bottom), ``RL`` (Right->Left), ``BT`` (Bottom->Top) +dag_orientation = LR + +# Puts the webserver in demonstration mode; blurs the names of Operators for +# privacy. +demo_mode = False + +# The amount of time (in secs) webserver will wait for initial handshake +# while fetching logs from other worker machine +log_fetch_timeout_sec = 5 + +# Time interval (in secs) to wait before next log fetching. +log_fetch_delay_sec = 2 + +# Distance away from page bottom to enable auto tailing. +log_auto_tailing_offset = 30 + +# Animation speed for auto tailing log display. +log_animation_speed = 1000 + +# By default, the webserver shows paused DAGs. Flip this to hide paused +# DAGs by default +hide_paused_dags_by_default = False + +# Consistent page size across all listing views in the UI +page_size = 100 + +# Define the color of navigation bar +navbar_color = #fff + +# Default dagrun to show in UI +default_dag_run_display_number = 25 + +# Enable werkzeug ``ProxyFix`` middleware for reverse proxy +enable_proxy_fix = False + +# Number of values to trust for ``X-Forwarded-For``. +# More info: https://werkzeug.palletsprojects.com/en/0.16.x/middleware/proxy_fix/ +proxy_fix_x_for = 1 + +# Number of values to trust for ``X-Forwarded-Proto`` +proxy_fix_x_proto = 1 + +# Number of values to trust for ``X-Forwarded-Host`` +proxy_fix_x_host = 1 + +# Number of values to trust for ``X-Forwarded-Port`` +proxy_fix_x_port = 1 + +# Number of values to trust for ``X-Forwarded-Prefix`` +proxy_fix_x_prefix = 1 + +# Set secure flag on session cookie +cookie_secure = False + +# Set samesite policy on session cookie +cookie_samesite = Lax + +# Default setting for wrap toggle on DAG code and TI log views. +default_wrap = False + +# Allow the UI to be rendered in a frame +x_frame_enabled = True + +# Send anonymous user activity to your analytics tool +# choose from google_analytics, segment, or metarouter +# analytics_tool = + +# Unique ID of your account in the analytics tool +# analytics_id = + +# 'Recent Tasks' stats will show for old DagRuns if set +show_recent_stats_for_completed_runs = True + +# Update FAB permissions and sync security manager roles +# on webserver startup +update_fab_perms = True + +# The UI cookie lifetime in minutes. User will be logged out from UI after +# ``session_lifetime_minutes`` of non-activity +session_lifetime_minutes = 43200 + +[email] + +# Configuration email backend and whether to +# send email alerts on retry or failure +# Email backend to use +email_backend = airflow.utils.email.send_email_smtp + +# Whether email alerts should be sent when a task is retried +default_email_on_retry = True + +# Whether email alerts should be sent when a task failed +default_email_on_failure = True + +[smtp] + +# If you want airflow to send emails on retries, failure, and you want to use +# the airflow.utils.email.send_email_smtp function, you have to configure an +# smtp server here +smtp_host = localhost +smtp_starttls = True +smtp_ssl = False +# Example: smtp_user = airflow +# smtp_user = +# Example: smtp_password = airflow +# smtp_password = +smtp_port = 25 +smtp_mail_from = airflow@example.com +smtp_timeout = 30 +smtp_retry_limit = 5 + +[sentry] + +# Sentry (https://docs.sentry.io) integration. Here you can supply +# additional configuration options based on the Python platform. See: +# https://docs.sentry.io/error-reporting/configuration/?platform=python. +# Unsupported options: ``integrations``, ``in_app_include``, ``in_app_exclude``, +# ``ignore_errors``, ``before_breadcrumb``, ``before_send``, ``transport``. +# Enable error reporting to Sentry +sentry_on = false +sentry_dsn = + +[celery_kubernetes_executor] + +# This section only applies if you are using the ``CeleryKubernetesExecutor`` in +# ``[core]`` section above +# Define when to send a task to ``KubernetesExecutor`` when using ``CeleryKubernetesExecutor``. +# When the queue of a task is ``kubernetes_queue``, the task is executed via ``KubernetesExecutor``, +# otherwise via ``CeleryExecutor`` +kubernetes_queue = kubernetes + +[celery] + +# This section only applies if you are using the CeleryExecutor in +# ``[core]`` section above +# The app name that will be used by celery +celery_app_name = airflow.executors.celery_executor + +# The concurrency that will be used when starting workers with the +# ``airflow celery worker`` command. This defines the number of task instances that +# a worker will take, so size up your workers based on the resources on +# your worker box and the nature of your tasks +worker_concurrency = 8 + +# The maximum and minimum concurrency that will be used when starting workers with the +# ``airflow celery worker`` command (always keep minimum processes, but grow +# to maximum if necessary). Note the value should be max_concurrency,min_concurrency +# Pick these numbers based on resources on worker box and the nature of the task. +# If autoscale option is available, worker_concurrency will be ignored. +# http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-autoscale +# Example: worker_autoscale = 16,12 +# worker_autoscale = + +# Used to increase the number of tasks that a worker prefetches which can improve performance. +# The number of processes multiplied by worker_prefetch_multiplier is the number of tasks +# that are prefetched by a worker. A value greater than 1 can result in tasks being unnecessarily +# blocked if there are multiple workers and one worker prefetches tasks that sit behind long +# running tasks while another worker has unutilized processes that are unable to process the already +# claimed blocked tasks. +# https://docs.celeryproject.org/en/stable/userguide/optimizing.html#prefetch-limits +# Example: worker_prefetch_multiplier = 1 +# worker_prefetch_multiplier = + +# When you start an airflow worker, airflow starts a tiny web server +# subprocess to serve the workers local log files to the airflow main +# web server, who then builds pages and sends them to users. This defines +# the port on which the logs are served. It needs to be unused, and open +# visible from the main web server to connect into the workers. +worker_log_server_port = 8793 + +# Umask that will be used when starting workers with the ``airflow celery worker`` +# in daemon mode. This control the file-creation mode mask which determines the initial +# value of file permission bits for newly created files. +worker_umask = 0o077 + +# The Celery broker URL. Celery supports RabbitMQ, Redis and experimentally +# a sqlalchemy database. Refer to the Celery documentation for more information. +broker_url = redis://redis:6379/0 + +# The Celery result_backend. When a job finishes, it needs to update the +# metadata of the job. Therefore it will post a message on a message bus, +# or insert it into a database (depending of the backend) +# This status is used by the scheduler to update the state of the task +# The use of a database is highly recommended +# http://docs.celeryproject.org/en/latest/userguide/configuration.html#task-result-backend-settings +result_backend = db+postgresql://postgres:airflow@postgres/airflow + +# Celery Flower is a sweet UI for Celery. Airflow has a shortcut to start +# it ``airflow celery flower``. This defines the IP that Celery Flower runs on +flower_host = 0.0.0.0 + +# The root URL for Flower +# Example: flower_url_prefix = /flower +flower_url_prefix = + +# This defines the port that Celery Flower runs on +flower_port = 5555 + +# Securing Flower with Basic Authentication +# Accepts user:password pairs separated by a comma +# Example: flower_basic_auth = user1:password1,user2:password2 +flower_basic_auth = + +# Default queue that tasks get assigned to and that worker listen on. +default_queue = default + +# How many processes CeleryExecutor uses to sync task state. +# 0 means to use max(1, number of cores - 1) processes. +sync_parallelism = 0 + +# Import path for celery configuration options +celery_config_options = airflow.config_templates.default_celery.DEFAULT_CELERY_CONFIG +ssl_active = False +ssl_key = +ssl_cert = +ssl_cacert = + +# Celery Pool implementation. +# Choices include: ``prefork`` (default), ``eventlet``, ``gevent`` or ``solo``. +# See: +# https://docs.celeryproject.org/en/latest/userguide/workers.html#concurrency +# https://docs.celeryproject.org/en/latest/userguide/concurrency/eventlet.html +pool = prefork + +# The number of seconds to wait before timing out ``send_task_to_executor`` or +# ``fetch_celery_task_state`` operations. +operation_timeout = 1.0 + +# Celery task will report its status as 'started' when the task is executed by a worker. +# This is used in Airflow to keep track of the running tasks and if a Scheduler is restarted +# or run in HA mode, it can adopt the orphan tasks launched by previous SchedulerJob. +task_track_started = True + +# Time in seconds after which Adopted tasks are cleared by CeleryExecutor. This is helpful to clear +# stalled tasks. +task_adoption_timeout = 600 + +# The Maximum number of retries for publishing task messages to the broker when failing +# due to ``AirflowTaskTimeout`` error before giving up and marking Task as failed. +task_publish_max_retries = 3 + +# Worker initialisation check to validate Metadata Database connection +worker_precheck = False + +[celery_broker_transport_options] + +# This section is for specifying options which can be passed to the +# underlying celery broker transport. See: +# http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-broker_transport_options +# The visibility timeout defines the number of seconds to wait for the worker +# to acknowledge the task before the message is redelivered to another worker. +# Make sure to increase the visibility timeout to match the time of the longest +# ETA you're planning to use. +# visibility_timeout is only supported for Redis and SQS celery brokers. +# See: +# http://docs.celeryproject.org/en/master/userguide/configuration.html#std:setting-broker_transport_options +# Example: visibility_timeout = 21600 +# visibility_timeout = + +[dask] + +# This section only applies if you are using the DaskExecutor in +# [core] section above +# The IP address and port of the Dask cluster's scheduler. +cluster_address = 127.0.0.1:8786 + +# TLS/ SSL settings to access a secured Dask scheduler. +tls_ca = +tls_cert = +tls_key = + +[scheduler] +# Task instances listen for external kill signal (when you clear tasks +# from the CLI or the UI), this defines the frequency at which they should +# listen (in seconds). +job_heartbeat_sec = 5 + +# How often (in seconds) to check and tidy up 'running' TaskInstancess +# that no longer have a matching DagRun +clean_tis_without_dagrun_interval = 15.0 + +# The scheduler constantly tries to trigger new tasks (look at the +# scheduler section in the docs for more information). This defines +# how often the scheduler should run (in seconds). +scheduler_heartbeat_sec = 5 + +# The number of times to try to schedule each DAG file +# -1 indicates unlimited number +num_runs = -1 + +# The number of seconds to wait between consecutive DAG file processing +processor_poll_interval = 1 + +# after how much time (seconds) a new DAGs should be picked up from the filesystem +min_file_process_interval = 0 + +# How often (in seconds) to scan the DAGs directory for new files. Default to 5 minutes. +dag_dir_list_interval = 300 + +# How often should stats be printed to the logs. Setting to 0 will disable printing stats +print_stats_interval = 30 + +# How often (in seconds) should pool usage stats be sent to statsd (if statsd_on is enabled) +pool_metrics_interval = 5.0 + +# If the last scheduler heartbeat happened more than scheduler_health_check_threshold +# ago (in seconds), scheduler is considered unhealthy. +# This is used by the health check in the "/health" endpoint +scheduler_health_check_threshold = 30 + +# How often (in seconds) should the scheduler check for orphaned tasks and SchedulerJobs +orphaned_tasks_check_interval = 300.0 +child_process_log_directory = /usr/local/airflow/logs/scheduler + +# Local task jobs periodically heartbeat to the DB. If the job has +# not heartbeat in this many seconds, the scheduler will mark the +# associated task instance as failed and will re-schedule the task. +scheduler_zombie_task_threshold = 300 + +# Turn off scheduler catchup by setting this to ``False``. +# Default behavior is unchanged and +# Command Line Backfills still work, but the scheduler +# will not do scheduler catchup if this is ``False``, +# however it can be set on a per DAG basis in the +# DAG definition (catchup) +catchup_by_default = True + +# This changes the batch size of queries in the scheduling main loop. +# If this is too high, SQL query performance may be impacted by one +# or more of the following: +# - reversion to full table scan +# - complexity of query predicate +# - excessive locking +# Additionally, you may hit the maximum allowable query length for your db. +# Set this to 0 for no limit (not advised) +max_tis_per_query = 512 + +# Should the scheduler issue ``SELECT ... FOR UPDATE`` in relevant queries. +# If this is set to False then you should not run more than a single +# scheduler at once +use_row_level_locking = True + +# Max number of DAGs to create DagRuns for per scheduler loop +# +# Default: 10 +# max_dagruns_to_create_per_loop = + +# How many DagRuns should a scheduler examine (and lock) when scheduling +# and queuing tasks. +# +# Default: 20 +# max_dagruns_per_loop_to_schedule = + +# Should the Task supervisor process perform a "mini scheduler" to attempt to schedule more tasks of the +# same DAG. Leaving this on will mean tasks in the same DAG execute quicker, but might starve out other +# dags in some circumstances +# +# Default: True +# schedule_after_task_execution = + +# The scheduler can run multiple processes in parallel to parse dags. +# This defines how many processes will run. +parsing_processes = 2 + +# Turn off scheduler use of cron intervals by setting this to False. +# DAGs submitted manually in the web UI or with trigger_dag will still run. +use_job_schedule = True + +# Allow externally triggered DagRuns for Execution Dates in the future +# Only has effect if schedule_interval is set to None in DAG +allow_trigger_in_future = False + +[kerberos] +ccache = /tmp/airflow_krb5_ccache + +# gets augmented with fqdn +principal = airflow +reinit_frequency = 3600 +kinit_path = kinit +keytab = airflow.keytab + +[github_enterprise] +api_rev = v3 + +[admin] +# UI to hide sensitive variable fields when set to True +hide_sensitive_variable_fields = True + +# A comma-separated list of sensitive keywords to look for in variables names. +sensitive_variable_fields = + +[elasticsearch] +# Elasticsearch host +host = + +# Format of the log_id, which is used to query for a given tasks logs +log_id_template = {dag_id}-{task_id}-{execution_date}-{try_number} + +# Used to mark the end of a log stream for a task +end_of_log_mark = end_of_log + +# Qualified URL for an elasticsearch frontend (like Kibana) with a template argument for log_id +# Code will construct log_id using the log_id template from the argument above. +# NOTE: The code will prefix the https:// automatically, don't include that here. +frontend = + +# Write the task logs to the stdout of the worker, rather than the default files +write_stdout = False + +# Instead of the default log formatter, write the log lines as JSON +json_format = False + +# Log fields to also attach to the json output, if enabled +json_fields = asctime, filename, lineno, levelname, message + +[elasticsearch_configs] +use_ssl = False +verify_certs = True + +[kubernetes] +# Path to the YAML pod file. If set, all other kubernetes-related fields are ignored. +pod_template_file = + +# The repository of the Kubernetes Image for the Worker to Run +worker_container_repository = + +# The tag of the Kubernetes Image for the Worker to Run +worker_container_tag = + +# The Kubernetes namespace where airflow workers should be created. Defaults to ``default`` +namespace = default + +# If True, all worker pods will be deleted upon termination +delete_worker_pods = True + +# If False (and delete_worker_pods is True), +# failed worker pods will not be deleted so users can investigate them. +delete_worker_pods_on_failure = False + +# Number of Kubernetes Worker Pod creation calls per scheduler loop. +# Note that the current default of "1" will only launch a single pod +# per-heartbeat. It is HIGHLY recommended that users increase this +# number to match the tolerance of their kubernetes cluster for +# better performance. +worker_pods_creation_batch_size = 1 + +# Allows users to launch pods in multiple namespaces. +# Will require creating a cluster-role for the scheduler +multi_namespace_mode = False + +# Use the service account kubernetes gives to pods to connect to kubernetes cluster. +# It's intended for clients that expect to be running inside a pod running on kubernetes. +# It will raise an exception if called from a process not running in a kubernetes environment. +in_cluster = True + +# When running with in_cluster=False change the default cluster_context or config_file +# options to Kubernetes client. Leave blank these to use default behaviour like ``kubectl`` has. +# cluster_context = + +# Path to the kubernetes configfile to be used when ``in_cluster`` is set to False +# config_file = + +# Keyword parameters to pass while calling a kubernetes client core_v1_api methods +# from Kubernetes Executor provided as a single line formatted JSON dictionary string. +# List of supported params are similar for all core_v1_apis, hence a single config +# variable for all apis. See: +# https://raw.githubusercontent.com/kubernetes-client/python/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/api/core_v1_api.py +kube_client_request_args = + +# Optional keyword arguments to pass to the ``delete_namespaced_pod`` kubernetes client +# ``core_v1_api`` method when using the Kubernetes Executor. +# This should be an object and can contain any of the options listed in the ``v1DeleteOptions`` +# class defined here: +# https://github.com/kubernetes-client/python/blob/41f11a09995efcd0142e25946adc7591431bfb2f/kubernetes/client/models/v1_delete_options.py#L19 +# Example: delete_option_kwargs = {"grace_period_seconds": 10} +delete_option_kwargs = + +# Enables TCP keepalive mechanism. This prevents Kubernetes API requests to hang indefinitely +# when idle connection is time-outed on services like cloud load balancers or firewalls. +enable_tcp_keepalive = False + +# When the `enable_tcp_keepalive` option is enabled, TCP probes a connection that has +# been idle for `tcp_keep_idle` seconds. +tcp_keep_idle = 120 + +# When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond +# to a keepalive probe, TCP retransmits the probe after `tcp_keep_intvl` seconds. +tcp_keep_intvl = 30 + +# When the `enable_tcp_keepalive` option is enabled, if Kubernetes API does not respond +# to a keepalive probe, TCP retransmits the probe `tcp_keep_cnt number` of times before +# a connection is considered to be broken. +tcp_keep_cnt = 6 + +[smart_sensor] +# When `use_smart_sensor` is True, Airflow redirects multiple qualified sensor tasks to +# smart sensor task. +use_smart_sensor = False + +# `shard_code_upper_limit` is the upper limit of `shard_code` value. The `shard_code` is generated +# by `hashcode % shard_code_upper_limit`. +shard_code_upper_limit = 10000 + +# The number of running smart sensor processes for each service. +shards = 5 + +# comma separated sensor classes support in smart_sensor. +sensors_enabled = NamedHivePartitionSensor diff --git a/dags/treino01.py b/dags/t01.py similarity index 95% rename from dags/treino01.py rename to dags/t01.py index 550e457e..617c2e11 100644 --- a/dags/treino01.py +++ b/dags/t01.py @@ -1,41 +1,41 @@ -from airflow import DAG -from airflow.operators.bash_operator import BashOperator -from airflow.operators.python_operator import PythonOperator -from datetime import datetime, timedelta - -default_args = { - 'owner': 'Neylson Crepalde', - "depends_on_past": False, - "start_date": datetime(2020, 11, 14), - "email": ["airflow@airflow.com"], - "email_on_failure": False, - "email_on_retry": False, - "retries": 1, - "retry_delay": timedelta(minutes=2), -} - -dag = DAG( - "treino-01", - description="Básico de Bash operators e Python operators", - default_args=default_args, - schedule_interval=timedelta(minutes=1) -) - -def hello_word(): - print("Hello Airflow from Python") - - -hello_bash = BashOperator( - task_id="hello-bash", - bash_command='echo "Hello Airflow from bash"', - dag=dag -) - -hello_python = PythonOperator( - task_id='hello-python', - python_callable=hello_word, - dag=dag -) - - +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.python_operator import PythonOperator +from datetime import datetime, timedelta + +default_args = { + 'owner': 'Neylson Crepalde', + "depends_on_past": False, + "start_date": datetime(2020, 11, 14), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=2), +} + +dag = DAG( + "treino-01", + description="Básico de Bash operators e Python operators", + default_args=default_args, + schedule_interval=timedelta(minutes=1) +) + +def hello_word(): + print("Hello Airflow from Python") + + +hello_bash = BashOperator( + task_id="hello-bash", + bash_command='echo "Hello Airflow from bash"', + dag=dag +) + +hello_python = PythonOperator( + task_id='hello-python', + python_callable=hello_word, + dag=dag +) + + hello_bash >> hello_python \ No newline at end of file diff --git a/dags/treino02.py b/dags/treino02.py index a8903b8d..6382632f 100644 --- a/dags/treino02.py +++ b/dags/treino02.py @@ -1,53 +1,53 @@ -from airflow import DAG -from airflow.operators.bash_operator import BashOperator -from airflow.operators.python_operator import PythonOperator -from datetime import datetime, timedelta -import pandas as pd - -default_args = { - 'owner': 'Neylson Crepalde', - "depends_on_past": False, - "start_date": datetime(2020, 11, 14, 23, 50), - "email": ["airflow@airflow.com"], - "email_on_failure": False, - "email_on_retry": False, - "retries": 1, - "retry_delay": timedelta(minutes=1), -} - -dag = DAG( - "treino-02", - description="Get Titanic data from internet and calculate mean age", - default_args=default_args, - schedule_interval=timedelta(minutes=5) -) - -get_data = BashOperator( - task_id="get-data", - bash_command='curl https://raw.githubusercontent.com/A3Data/hermione/master/hermione/file_text/train.csv -o ~/train.csv', - dag=dag -) - -def calculate_mean_age(): - df = pd.read_csv('~/train.csv') - med = df.Age.mean() - return med - -task_calculate_mean = PythonOperator( - task_id='calculate-mean-age', - python_callable=calculate_mean_age, - dag=dag -) - -def print_age(**context): - value = context['task_instance'].xcom_pull(task_ids='calculate-mean-age') - print(f"Mean age is {value}") - -task_mean_age = PythonOperator( - task_id="say-mean-age", - python_callable=print_age, - provide_context=True, - dag=dag -) - +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.python_operator import PythonOperator +from datetime import datetime, timedelta +import pandas as pd + +default_args = { + 'owner': 'Neylson Crepalde', + "depends_on_past": False, + "start_date": datetime(2020, 11, 14, 23, 50), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=1), +} + +dag = DAG( + "treino-02", + description="Get Titanic data from internet and calculate mean age", + default_args=default_args, + schedule_interval=timedelta(minutes=5) +) + +get_data = BashOperator( + task_id="get-data", + bash_command='curl https://raw.githubusercontent.com/A3Data/hermione/master/hermione/file_text/train.csv -o ~/train.csv', + dag=dag +) + +def calculate_mean_age(): + df = pd.read_csv('~/train.csv') + med = df.Age.mean() + return med + +task_calculate_mean = PythonOperator( + task_id='calculate-mean-age', + python_callable=calculate_mean_age, + dag=dag +) + +def print_age(**context): + value = context['task_instance'].xcom_pull(task_ids='calculate-mean-age') + print(f"Mean age is {value}") + +task_mean_age = PythonOperator( + task_id="say-mean-age", + python_callable=print_age, + provide_context=True, + dag=dag +) + get_data >> task_calculate_mean >> task_mean_age \ No newline at end of file diff --git a/dags/treino03.py b/dags/treino03.py index e2e2f23d..df1ff42b 100644 --- a/dags/treino03.py +++ b/dags/treino03.py @@ -1,80 +1,80 @@ -from airflow import DAG -from airflow.operators.bash_operator import BashOperator -from airflow.operators.python_operator import PythonOperator, BranchPythonOperator -from datetime import datetime, timedelta -import pandas as pd -import random - -default_args = { - 'owner': 'Neylson Crepalde', - "depends_on_past": False, - "start_date": datetime(2020, 11, 15, 16, 20), - "email": ["airflow@airflow.com"], - "email_on_failure": False, - "email_on_retry": False, - "retries": 1, - "retry_delay": timedelta(minutes=1), -} - -dag = DAG( - "treino-03", - description="Pega dados do Titanic e calcula idade média para homens ou mulheres", - default_args=default_args, - schedule_interval=timedelta(minutes=2) -) - -get_data = BashOperator( - task_id="get-data", - bash_command='curl https://raw.githubusercontent.com/A3Data/hermione/master/hermione/file_text/train.csv -o /usr/local/airflow/train.csv', - dag=dag -) - - -def sorteia_h_m(): - return random.choice(['male', 'female']) - -escolhe_h_m = PythonOperator( - task_id='escolhe-h-m', - python_callable=sorteia_h_m, - dag=dag -) - -def MouF(**context): - value = context['task_instance'].xcom_pull(task_ids='escolhe-h-m') - if value == 'male': - return 'branch_homem' - if value == 'female': - return 'branch_mulher' - -male_female = BranchPythonOperator( - task_id='condicional', - python_callable=MouF, - provide_context=True, - dag=dag -) - - -def mean_homen(): - df = pd.read_csv('/usr/local/airflow/train.csv') - df = df.loc[df.Sex == 'male'] - print(f"Média de idade dos homens no Titanic: {df.Age.mean()}") - -branch_homem = PythonOperator( - task_id='branch_homem', - python_callable=mean_homen, - dag=dag -) - -def mean_mulher(): - df = pd.read_csv('/usr/local/airflow/train.csv') - df = df.loc[df.Sex == 'female'] - print(f"Média de idade das mulheres no Titanic: {df.Age.mean()}") - -branch_mulher = PythonOperator( - task_id='branch_mulher', - python_callable=mean_mulher, - dag=dag -) - - +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.python_operator import PythonOperator, BranchPythonOperator +from datetime import datetime, timedelta +import pandas as pd +import random + +default_args = { + 'owner': 'Neylson Crepalde', + "depends_on_past": False, + "start_date": datetime(2020, 11, 15, 16, 20), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=1), +} + +dag = DAG( + "treino-03", + description="Pega dados do Titanic e calcula idade média para homens ou mulheres", + default_args=default_args, + schedule_interval=timedelta(minutes=2) +) + +get_data = BashOperator( + task_id="get-data", + bash_command='curl https://raw.githubusercontent.com/A3Data/hermione/master/hermione/file_text/train.csv -o /usr/local/airflow/train.csv', + dag=dag +) + + +def sorteia_h_m(): + return random.choice(['male', 'female']) + +escolhe_h_m = PythonOperator( + task_id='escolhe-h-m', + python_callable=sorteia_h_m, + dag=dag +) + +def MouF(**context): + value = context['task_instance'].xcom_pull(task_ids='escolhe-h-m') + if value == 'male': + return 'branch_homem' + if value == 'female': + return 'branch_mulher' + +male_female = BranchPythonOperator( + task_id='condicional', + python_callable=MouF, + provide_context=True, + dag=dag +) + + +def mean_homen(): + df = pd.read_csv('/usr/local/airflow/train.csv') + df = df.loc[df.Sex == 'male'] + print(f"Média de idade dos homens no Titanic: {df.Age.mean()}") + +branch_homem = PythonOperator( + task_id='branch_homem', + python_callable=mean_homen, + dag=dag +) + +def mean_mulher(): + df = pd.read_csv('/usr/local/airflow/train.csv') + df = df.loc[df.Sex == 'female'] + print(f"Média de idade das mulheres no Titanic: {df.Age.mean()}") + +branch_mulher = PythonOperator( + task_id='branch_mulher', + python_callable=mean_mulher, + dag=dag +) + + get_data >> escolhe_h_m >> male_female >> [branch_homem, branch_mulher] \ No newline at end of file diff --git a/dags/treino04.py b/dags/treino04.py index f7780d9b..f91f6496 100644 --- a/dags/treino04.py +++ b/dags/treino04.py @@ -1,85 +1,85 @@ -from airflow import DAG -from airflow.operators.bash_operator import BashOperator -from airflow.operators.python_operator import PythonOperator, BranchPythonOperator -from datetime import datetime, timedelta -import zipfile -import random -import pandas as pd - -default_args = { - 'owner': 'Neylson Crepalde', - "depends_on_past": False, - "start_date": datetime(2020, 12, 30, 18, 10), - "email": ["airflow@airflow.com"], - "email_on_failure": False, - "email_on_retry": False - #"retries": 1, - #"retry_delay": timedelta(minutes=1), -} - -dag = DAG( - "treino-04", - description="Uma dag com condicionais", - default_args=default_args, - schedule_interval=timedelta(minutes=2) -) - -get_data = BashOperator( - task_id="get-data", - bash_command='curl https://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', - trigger_rule="all_done", - dag=dag -) - - -def unzip_file(): - with zipfile.ZipFile("/usr/local/airflow/data/microdados_enade_2019.zip", 'r') as zipped: - zipped.extractall("/usr/local/airflow/data") - -unzip_data = PythonOperator( - task_id='unzip-data', - python_callable=unzip_file, - dag=dag -) - - -def select_student(): - df = pd.read_csv('/usr/local/airflow/data/microdados_enade_2019/2019/3.DADOS/microdados_enade_2019.txt', sep=';', decimal=',') - escolha = random.randint(0, df.shape[0]-1) - aluno = df.iloc[escolha] - return aluno.TP_SEXO - -pick_student = PythonOperator( - task_id="pick-student", - python_callable=select_student, - dag=dag -) - -def MouF(**context): - value = context['task_instance'].xcom_pull(task_ids='pick-student') - if value == 'M': - return 'male_branch' - elif value == 'F': - return 'female_branch' - -male_of_female = BranchPythonOperator( - task_id='condition-male_or_female', - python_callable=MouF, - provide_context=True, - dag=dag -) - - -male_branch = BashOperator( - task_id="male_branch", - bash_command='echo "Estudante escolhido foi do sexo Masculino"', - dag=dag -) - -female_branch = BashOperator( - task_id="female_branch", - bash_command='echo "Estudante escolhido foi do sexo Feminino"', - dag=dag -) - +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.python_operator import PythonOperator, BranchPythonOperator +from datetime import datetime, timedelta +import zipfile +import random +import pandas as pd + +default_args = { + 'owner': 'Neylson Crepalde', + "depends_on_past": False, + "start_date": datetime(2020, 12, 30, 18, 10), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False + #"retries": 1, + #"retry_delay": timedelta(minutes=1), +} + +dag = DAG( + "treino-04", + description="Uma dag com condicionais", + default_args=default_args, + schedule_interval=timedelta(minutes=2) +) + +get_data = BashOperator( + task_id="get-data", + bash_command='curl https://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', + trigger_rule="all_done", + dag=dag +) + + +def unzip_file(): + with zipfile.ZipFile("/usr/local/airflow/data/microdados_enade_2019.zip", 'r') as zipped: + zipped.extractall("/usr/local/airflow/data") + +unzip_data = PythonOperator( + task_id='unzip-data', + python_callable=unzip_file, + dag=dag +) + + +def select_student(): + df = pd.read_csv('/usr/local/airflow/data/microdados_enade_2019/2019/3.DADOS/microdados_enade_2019.txt', sep=';', decimal=',') + escolha = random.randint(0, df.shape[0]-1) + aluno = df.iloc[escolha] + return aluno.TP_SEXO + +pick_student = PythonOperator( + task_id="pick-student", + python_callable=select_student, + dag=dag +) + +def MouF(**context): + value = context['task_instance'].xcom_pull(task_ids='pick-student') + if value == 'M': + return 'male_branch' + elif value == 'F': + return 'female_branch' + +male_of_female = BranchPythonOperator( + task_id='condition-male_or_female', + python_callable=MouF, + provide_context=True, + dag=dag +) + + +male_branch = BashOperator( + task_id="male_branch", + bash_command='echo "Estudante escolhido foi do sexo Masculino"', + dag=dag +) + +female_branch = BashOperator( + task_id="female_branch", + bash_command='echo "Estudante escolhido foi do sexo Feminino"', + dag=dag +) + get_data >> unzip_data >> pick_student >> male_of_female >> [male_branch, female_branch] \ No newline at end of file diff --git a/dags/treino05.py b/dags/treino05.py index 7636e6fc..596a14e3 100644 --- a/dags/treino05.py +++ b/dags/treino05.py @@ -1,208 +1,208 @@ -from airflow import DAG -from airflow.operators.bash_operator import BashOperator -from airflow.operators.python_operator import PythonOperator, BranchPythonOperator -from datetime import datetime, timedelta -import zipfile -import pandas as pd - -data_path = '/usr/local/airflow/data/microdados_enade_2019/2019/3.DADOS/' -arquivo = data_path + 'microdados_enade_2019.txt' - -default_args = { - 'owner': 'Neylson Crepalde', - "depends_on_past": False, - "start_date": datetime(2020, 12, 30, 18, 10), - "email": ["airflow@airflow.com"], - "email_on_failure": False, - "email_on_retry": False - #"retries": 1, - #"retry_delay": timedelta(minutes=1), -} - -dag = DAG( - "treino-05", - description="Uma dag com vários paralelismos", - default_args=default_args, - schedule_interval="*/3 * * * *" -) - -get_data = BashOperator( - task_id="get-data", - bash_command='curl https://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', - trigger_rule="all_done", - dag=dag -) - -def unzip_file(): - with zipfile.ZipFile("/usr/local/airflow/data/microdados_enade_2019.zip", 'r') as zipped: - zipped.extractall("/usr/local/airflow/data") - -unzip_data = PythonOperator( - task_id='unzip-data', - python_callable=unzip_file, - dag=dag -) - - -def aplica_filtros(): - cols = ['CO_GRUPO', 'TP_SEXO', 'NU_IDADE', 'NT_GER', 'NT_FG', 'NT_CE', - 'QE_I01','QE_I02','QE_I04','QE_I05','QE_I08'] - enade = pd.read_csv(arquivo, sep=';', decimal=',', usecols=cols) - enade = enade.loc[ - (enade.NU_IDADE > 20) & - (enade.NU_IDADE < 40) & - (enade.NT_GER > 0) - ] - enade.to_csv(data_path + 'enade_filtrado.csv', index=False) - -task_aplica_filtro = PythonOperator( - task_id="aplica_filtro", - python_callable=aplica_filtros, - dag=dag -) - -def constroi_idade_centralizada(): - idade = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['NU_IDADE']) - idade['idadecent'] = idade.NU_IDADE - idade.NU_IDADE.mean() - idade[['idadecent']].to_csv(data_path + "idadecent.csv", index=False) - -def constroi_idade_cent_quad(): - idadecent = pd.read_csv(data_path + "idadecent.csv") - idadecent['idade2'] = idadecent.idadecent ** 2 - idadecent[['idade2']].to_csv(data_path + 'idadequadrado.csv', index=False) - -task_idade_cent = PythonOperator( - task_id='constroi_idade_centralizada', - python_callable=constroi_idade_centralizada, - dag=dag -) - -task_idade_quad = PythonOperator( - task_id='constroi_idade_ao_quadrado', - python_callable=constroi_idade_cent_quad, - dag=dag -) - -def constroi_est_civil(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I01']) - filtro['estcivil'] = filtro.QE_I01.replace({ - 'A': 'Solteiro', - 'B': 'Casado', - 'C': 'Separado', - 'D': 'Viúvo', - 'E': 'Outro' - }) - filtro[['estcivil']].to_csv(data_path + 'estcivil.csv', index=False) - -task_est_civil = PythonOperator( - task_id='constroi_est_civil', - python_callable=constroi_est_civil, - dag=dag -) - -def constroi_cor(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I02']) - filtro['cor'] = filtro.QE_I02.replace({ - 'A': 'Branca', - 'B': 'Preta', - 'C': 'Amarela', - 'D': 'Parda', - 'E': 'Indígena', - 'F': "", - ' ': "" - }) - filtro[['cor']].to_csv(data_path + 'cor.csv', index=False) - -task_cor = PythonOperator( - task_id='constroi_cor_da_pele', - python_callable=constroi_cor, - dag=dag -) - -def constroi_escopai(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I04']) - filtro['escopai'] = filtro.QE_I04.replace({ - 'A': 0, - 'B': 1, - 'C': 2, - 'D': 3, - 'E': 4, - 'F': 5 - }) - filtro[['escopai']].to_csv(data_path + 'escopai.csv', index=False) - -task_escopai = PythonOperator( - task_id='constroi_escopai', - python_callable=constroi_escopai, - dag=dag -) - -def constroi_escomae(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I05']) - filtro['escomae'] = filtro.QE_I05.replace({ - 'A': 0, - 'B': 1, - 'C': 2, - 'D': 3, - 'E': 4, - 'F': 5 - }) - filtro[['escomae']].to_csv(data_path + 'escomae.csv', index=False) - -task_escomae = PythonOperator( - task_id='constroi_escomae', - python_callable=constroi_escomae, - dag=dag -) - -def constroi_renda(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I08']) - filtro['renda'] = filtro.QE_I08.replace({ - 'A': 0, - 'B': 1, - 'C': 2, - 'D': 3, - 'E': 4, - 'F': 5, - 'G': 6 - }) - filtro[['renda']].to_csv(data_path + 'renda.csv', index=False) - -task_renda = PythonOperator( - task_id='constroi_renda', - python_callable=constroi_renda, - dag=dag -) - -def join_data(): - filtro = pd.read_csv(data_path + 'enade_filtrado.csv') - idadecent = pd.read_csv(data_path + 'idadecent.csv') - idadequadrado = pd.read_csv(data_path + 'idadequadrado.csv') - estcivil = pd.read_csv(data_path + 'estcivil.csv') - cor = pd.read_csv(data_path + 'cor.csv') - escopai = pd.read_csv(data_path + 'escopai.csv') - escomae = pd.read_csv(data_path + 'escomae.csv') - renda = pd.read_csv(data_path + 'renda.csv') - - final = pd.concat([filtro, idadecent, idadequadrado, estcivil, cor, - escopai, escomae, renda], axis=1) - final.to_csv(data_path + 'enade_tratado.csv', index=False) - print(final) - -task_join = PythonOperator( - task_id='join_data', - python_callable=join_data, - dag=dag -) - - - -get_data >> unzip_data >> task_aplica_filtro >> [ - task_idade_cent, task_est_civil, task_cor ,task_escopai, task_escomae, task_renda -] -task_idade_quad.set_upstream(task_idade_cent) -task_join.set_upstream([ - task_idade_cent, task_est_civil, task_escopai, task_cor, task_escomae, task_renda, task_idade_quad -]) - - +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from airflow.operators.python_operator import PythonOperator, BranchPythonOperator +from datetime import datetime, timedelta +import zipfile +import pandas as pd + +data_path = '/usr/local/airflow/data/microdados_enade_2019/2019/3.DADOS/' +arquivo = data_path + 'microdados_enade_2019.txt' + +default_args = { + 'owner': 'Neylson Crepalde', + "depends_on_past": False, + "start_date": datetime(2020, 12, 30, 18, 10), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False + #"retries": 1, + #"retry_delay": timedelta(minutes=1), +} + +dag = DAG( + "treino-05", + description="Uma dag com vários paralelismos", + default_args=default_args, + schedule_interval="*/3 * * * *" +) + +get_data = BashOperator( + task_id="get-data", + bash_command='curl https://download.inep.gov.br/microdados/Enade_Microdados/microdados_enade_2019.zip -o /usr/local/airflow/data/microdados_enade_2019.zip', + trigger_rule="all_done", + dag=dag +) + +def unzip_file(): + with zipfile.ZipFile("/usr/local/airflow/data/microdados_enade_2019.zip", 'r') as zipped: + zipped.extractall("/usr/local/airflow/data") + +unzip_data = PythonOperator( + task_id='unzip-data', + python_callable=unzip_file, + dag=dag +) + + +def aplica_filtros(): + cols = ['CO_GRUPO', 'TP_SEXO', 'NU_IDADE', 'NT_GER', 'NT_FG', 'NT_CE', + 'QE_I01','QE_I02','QE_I04','QE_I05','QE_I08'] + enade = pd.read_csv(arquivo, sep=';', decimal=',', usecols=cols) + enade = enade.loc[ + (enade.NU_IDADE > 20) & + (enade.NU_IDADE < 40) & + (enade.NT_GER > 0) + ] + enade.to_csv(data_path + 'enade_filtrado.csv', index=False) + +task_aplica_filtro = PythonOperator( + task_id="aplica_filtro", + python_callable=aplica_filtros, + dag=dag +) + +def constroi_idade_centralizada(): + idade = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['NU_IDADE']) + idade['idadecent'] = idade.NU_IDADE - idade.NU_IDADE.mean() + idade[['idadecent']].to_csv(data_path + "idadecent.csv", index=False) + +def constroi_idade_cent_quad(): + idadecent = pd.read_csv(data_path + "idadecent.csv") + idadecent['idade2'] = idadecent.idadecent ** 2 + idadecent[['idade2']].to_csv(data_path + 'idadequadrado.csv', index=False) + +task_idade_cent = PythonOperator( + task_id='constroi_idade_centralizada', + python_callable=constroi_idade_centralizada, + dag=dag +) + +task_idade_quad = PythonOperator( + task_id='constroi_idade_ao_quadrado', + python_callable=constroi_idade_cent_quad, + dag=dag +) + +def constroi_est_civil(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I01']) + filtro['estcivil'] = filtro.QE_I01.replace({ + 'A': 'Solteiro', + 'B': 'Casado', + 'C': 'Separado', + 'D': 'Viúvo', + 'E': 'Outro' + }) + filtro[['estcivil']].to_csv(data_path + 'estcivil.csv', index=False) + +task_est_civil = PythonOperator( + task_id='constroi_est_civil', + python_callable=constroi_est_civil, + dag=dag +) + +def constroi_cor(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I02']) + filtro['cor'] = filtro.QE_I02.replace({ + 'A': 'Branca', + 'B': 'Preta', + 'C': 'Amarela', + 'D': 'Parda', + 'E': 'Indígena', + 'F': "", + ' ': "" + }) + filtro[['cor']].to_csv(data_path + 'cor.csv', index=False) + +task_cor = PythonOperator( + task_id='constroi_cor_da_pele', + python_callable=constroi_cor, + dag=dag +) + +def constroi_escopai(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I04']) + filtro['escopai'] = filtro.QE_I04.replace({ + 'A': 0, + 'B': 1, + 'C': 2, + 'D': 3, + 'E': 4, + 'F': 5 + }) + filtro[['escopai']].to_csv(data_path + 'escopai.csv', index=False) + +task_escopai = PythonOperator( + task_id='constroi_escopai', + python_callable=constroi_escopai, + dag=dag +) + +def constroi_escomae(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I05']) + filtro['escomae'] = filtro.QE_I05.replace({ + 'A': 0, + 'B': 1, + 'C': 2, + 'D': 3, + 'E': 4, + 'F': 5 + }) + filtro[['escomae']].to_csv(data_path + 'escomae.csv', index=False) + +task_escomae = PythonOperator( + task_id='constroi_escomae', + python_callable=constroi_escomae, + dag=dag +) + +def constroi_renda(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv', usecols=['QE_I08']) + filtro['renda'] = filtro.QE_I08.replace({ + 'A': 0, + 'B': 1, + 'C': 2, + 'D': 3, + 'E': 4, + 'F': 5, + 'G': 6 + }) + filtro[['renda']].to_csv(data_path + 'renda.csv', index=False) + +task_renda = PythonOperator( + task_id='constroi_renda', + python_callable=constroi_renda, + dag=dag +) + +def join_data(): + filtro = pd.read_csv(data_path + 'enade_filtrado.csv') + idadecent = pd.read_csv(data_path + 'idadecent.csv') + idadequadrado = pd.read_csv(data_path + 'idadequadrado.csv') + estcivil = pd.read_csv(data_path + 'estcivil.csv') + cor = pd.read_csv(data_path + 'cor.csv') + escopai = pd.read_csv(data_path + 'escopai.csv') + escomae = pd.read_csv(data_path + 'escomae.csv') + renda = pd.read_csv(data_path + 'renda.csv') + + final = pd.concat([filtro, idadecent, idadequadrado, estcivil, cor, + escopai, escomae, renda], axis=1) + final.to_csv(data_path + 'enade_tratado.csv', index=False) + print(final) + +task_join = PythonOperator( + task_id='join_data', + python_callable=join_data, + dag=dag +) + + + +get_data >> unzip_data >> task_aplica_filtro >> [ + task_idade_cent, task_est_civil, task_cor ,task_escopai, task_escomae, task_renda +] +task_idade_quad.set_upstream(task_idade_cent) +task_join.set_upstream([ + task_idade_cent, task_est_civil, task_escopai, task_cor, task_escomae, task_renda, task_idade_quad +]) + + diff --git a/dags/tuto.py b/dags/tuto.py index cead2b61..656203fd 100644 --- a/dags/tuto.py +++ b/dags/tuto.py @@ -1,48 +1,48 @@ -""" -Code that goes along with the Airflow located at: -http://airflow.readthedocs.org/en/latest/tutorial.html -""" -from airflow import DAG -from airflow.operators.bash_operator import BashOperator -from datetime import datetime, timedelta - - -default_args = { - "owner": "airflow", - "depends_on_past": False, - "start_date": datetime(2015, 6, 1), - "email": ["airflow@airflow.com"], - "email_on_failure": False, - "email_on_retry": False, - "retries": 1, - "retry_delay": timedelta(minutes=5), - # 'queue': 'bash_queue', - # 'pool': 'backfill', - # 'priority_weight': 10, - # 'end_date': datetime(2016, 1, 1), -} - -dag = DAG("tutorial", default_args=default_args, schedule_interval=timedelta(1)) - -# t1, t2 and t3 are examples of tasks created by instantiating operators -t1 = BashOperator(task_id="print_date", bash_command="date", dag=dag) - -t2 = BashOperator(task_id="sleep", bash_command="sleep 5", retries=3, dag=dag) - -templated_command = """ - {% for i in range(5) %} - echo "{{ ds }}" - echo "{{ macros.ds_add(ds, 7)}}" - echo "{{ params.my_param }}" - {% endfor %} -""" - -t3 = BashOperator( - task_id="templated", - bash_command=templated_command, - params={"my_param": "Parameter I passed in"}, - dag=dag, -) - -t2.set_upstream(t1) -t3.set_upstream(t1) +""" +Code that goes along with the Airflow located at: +http://airflow.readthedocs.org/en/latest/tutorial.html +""" +from airflow import DAG +from airflow.operators.bash_operator import BashOperator +from datetime import datetime, timedelta + + +default_args = { + "owner": "airflow", + "depends_on_past": False, + "start_date": datetime(2015, 6, 1), + "email": ["airflow@airflow.com"], + "email_on_failure": False, + "email_on_retry": False, + "retries": 1, + "retry_delay": timedelta(minutes=5), + # 'queue': 'bash_queue', + # 'pool': 'backfill', + # 'priority_weight': 10, + # 'end_date': datetime(2016, 1, 1), +} + +dag = DAG("tutorial", default_args=default_args, schedule_interval=timedelta(1)) + +# t1, t2 and t3 are examples of tasks created by instantiating operators +t1 = BashOperator(task_id="print_date", bash_command="date", dag=dag) + +t2 = BashOperator(task_id="sleep", bash_command="sleep 5", retries=3, dag=dag) + +templated_command = """ + {% for i in range(5) %} + echo "{{ ds }}" + echo "{{ macros.ds_add(ds, 7)}}" + echo "{{ params.my_param }}" + {% endfor %} +""" + +t3 = BashOperator( + task_id="templated", + bash_command=templated_command, + params={"my_param": "Parameter I passed in"}, + dag=dag, +) + +t2.set_upstream(t1) +t3.set_upstream(t1) diff --git a/data/.gitkeep b/data/.gitkeep index 8b137891..d3f5a12f 100644 --- a/data/.gitkeep +++ b/data/.gitkeep @@ -1 +1 @@ - + diff --git a/docker-compose-CeleryExecutor.yml b/docker-compose-CeleryExecutor.yml index 64ec7423..5edefc97 100644 --- a/docker-compose-CeleryExecutor.yml +++ b/docker-compose-CeleryExecutor.yml @@ -1,95 +1,98 @@ -version: '2.2' -services: - redis: - image: 'redis:5.0.5' - # command: redis-server --requirepass redispass - - postgres: - image: postgres:9.6 - environment: - - POSTGRES_USER=airflow - - POSTGRES_PASSWORD=airflow - - POSTGRES_DB=airflow - # Uncomment these lines to persist data on the local filesystem. - # - PGDATA=/var/lib/postgresql/data/pgdata - # volumes: - # - ./pgdata:/var/lib/postgresql/data/pgdata - - webserver: - image: neylsoncrepalde/airflow-docker:latest - restart: always - depends_on: - - postgres - - redis - environment: - - LOAD_EX=n - - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= - - EXECUTOR=Celery - # - POSTGRES_USER=airflow - # - POSTGRES_PASSWORD=airflow - # - POSTGRES_DB=airflow - # - REDIS_PASSWORD=redispass - volumes: - - ./dags:/usr/local/airflow/dags - - ./data:/usr/local/airflow/data - # Uncomment to include custom plugins - # - ./plugins:/usr/local/airflow/plugins - ports: - - "8080:8080" - command: webserver - healthcheck: - test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"] - interval: 30s - timeout: 30s - retries: 3 - - flower: - image: neylsoncrepalde/airflow-docker:latest - restart: always - depends_on: - - redis - environment: - - EXECUTOR=Celery - # - REDIS_PASSWORD=redispass - ports: - - "5555:5555" - command: celery flower - - scheduler: - image: neylsoncrepalde/airflow-docker:latest - restart: always - depends_on: - - webserver - volumes: - - ./dags:/usr/local/airflow/dags - - ./data:/usr/local/airflow/data - # Uncomment to include custom plugins - # - ./plugins:/usr/local/airflow/plugins - environment: - - LOAD_EX=n - - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= - - EXECUTOR=Celery - # - POSTGRES_USER=airflow - # - POSTGRES_PASSWORD=airflow - # - POSTGRES_DB=airflow - # - REDIS_PASSWORD=redispass - command: scheduler - - worker: - image: neylsoncrepalde/airflow-docker:latest - restart: always - depends_on: - - scheduler - volumes: - - ./dags:/usr/local/airflow/dags - - ./data:/usr/local/airflow/data - # Uncomment to include custom plugins - # - ./plugins:/usr/local/airflow/plugins - environment: - - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= - - EXECUTOR=Celery - # - POSTGRES_USER=airflow - # - POSTGRES_PASSWORD=airflow - # - POSTGRES_DB=airflow - # - REDIS_PASSWORD=redispass - command: celery worker +version: '2.2' +services: + redis: + image: 'redis:5.0.5' + # command: redis-server --requirepass redispass + + postgres: + image: postgres:9.6 + environment: + - POSTGRES_USER=airflow + - POSTGRES_PASSWORD=airflow + - POSTGRES_DB=airflow + # Uncomment these lines to persist data on the local filesystem. + # - PGDATA=/var/lib/postgresql/data/pgdata + # volumes: + # - ./pgdata:/var/lib/postgresql/data/pgdata + + webserver: + image: neylsoncrepalde/airflow-docker:latest + restart: always + depends_on: + - postgres + - redis + environment: + - LOAD_EX=n + - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= + - EXECUTOR=Celery + # - POSTGRES_USER=airflow + # - POSTGRES_PASSWORD=airflow + # - POSTGRES_DB=airflow + # - REDIS_PASSWORD=redispass + volumes: + - /D/Docker/volumes/airflow/dags:/usr/local/airflow/dags + - /D/Docker/volumes/airflow/data:/usr/local/airflow/data + - ./requirements.txt:/requirements.txt + # Uncomment to include custom plugins + # - ./plugins:/usr/local/airflow/plugins + ports: + - "8080:8080" + command: webserver + healthcheck: + test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"] + interval: 30s + timeout: 30s + retries: 3 + + flower: + image: neylsoncrepalde/airflow-docker:latest + restart: always + depends_on: + - redis + environment: + - EXECUTOR=Celery + # - REDIS_PASSWORD=redispass + ports: + - "5555:5555" + command: celery flower + + scheduler: + image: neylsoncrepalde/airflow-docker:latest + restart: always + depends_on: + - webserver + volumes: + - /D/Docker/volumes/airflow/dags:/usr/local/airflow/dags + - /D/Docker/volumes/airflow/data:/usr/local/airflow/data + - ./requirements.txt:/requirements.txt + # Uncomment to include custom plugins + # - ./plugins:/usr/local/airflow/plugins + environment: + - LOAD_EX=n + - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= + - EXECUTOR=Celery + # - POSTGRES_USER=airflow + # - POSTGRES_PASSWORD=airflow + # - POSTGRES_DB=airflow + # - REDIS_PASSWORD=redispass + command: scheduler + + worker: + image: neylsoncrepalde/airflow-docker:latest + restart: always + depends_on: + - scheduler + volumes: + - /D/Docker/volumes/airflow/dags:/usr/local/airflow/dags + - /D/Docker/volumes/airflow/data:/usr/local/airflow/data + - ./requirements.txt:/requirements.txt + # Uncomment to include custom plugins + # - ./plugins:/usr/local/airflow/plugins + environment: + - FERNET_KEY=46BKJoQYlPPOexq0OhDZnIlNepKFf87WFwLbfzqDDho= + - EXECUTOR=Celery + # - POSTGRES_USER=airflow + # - POSTGRES_PASSWORD=airflow + # - POSTGRES_DB=airflow + # - REDIS_PASSWORD=redispass + command: celery worker diff --git a/docker-compose-LocalExecutor.yml b/docker-compose-LocalExecutor.yml index 45466f7e..f9b60320 100644 --- a/docker-compose-LocalExecutor.yml +++ b/docker-compose-LocalExecutor.yml @@ -1,37 +1,38 @@ -version: '3.8' -services: - postgres: - image: postgres:9.6 - environment: - - POSTGRES_USER=airflow - - POSTGRES_PASSWORD=airflow - - POSTGRES_DB=airflow - logging: - options: - max-size: 10m - max-file: "3" - - webserver: - image: neylsoncrepalde/airflow-docker:latest - restart: always - depends_on: - - postgres - environment: - - LOAD_EX=n - - EXECUTOR=Local - logging: - options: - max-size: 10m - max-file: "3" - volumes: - - ./dags:/usr/local/airflow/dags - - ./data:/usr/local/airflow/data - # - ./plugins:/usr/local/airflow/plugins - ports: - - "8080:8080" - command: webserver - healthcheck: - test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"] - interval: 30s - timeout: 30s - retries: 3 +version: '3.8' +services: + postgres: + image: postgres:9.6 + environment: + - POSTGRES_USER=airflow + - POSTGRES_PASSWORD=airflow + - POSTGRES_DB=airflow + logging: + options: + max-size: 10m + max-file: "3" + + webserver: + image: neylsoncrepalde/airflow-docker:latest + restart: always + depends_on: + - postgres + environment: + - LOAD_EX=n + - EXECUTOR=Local + logging: + options: + max-size: 10m + max-file: "3" + volumes: + - /D/Docker/volumes/airflow/dags:/usr/local/airflow/dags + - /D/Docker/volumes/airflow/data:/usr/local/airflow/data + - ./requirements.txt:/requirements.txt + # - ./plugins:/usr/local/airflow/plugins + ports: + - "8080:8080" + command: webserver + healthcheck: + test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"] + interval: 30s + timeout: 30s + retries: 3 diff --git a/requirements.txt b/requirements.txt index 4ce062b4..2d59e0a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ -boto3 -pandas -pymongo +boto3 +pandas +pymongo +sqlalchemy +numpy +datetime \ No newline at end of file diff --git a/script/entrypoint.sh b/script/entrypoint.sh index 862c9d55..c6c0b120 100755 --- a/script/entrypoint.sh +++ b/script/entrypoint.sh @@ -1,136 +1,136 @@ -#!/usr/bin/env bash - -# User-provided configuration must always be respected. -# -# Therefore, this script must only derives Airflow AIRFLOW__ variables from other variables -# when the user did not provide their own configuration. - -TRY_LOOP="20" - -# Global defaults and back-compat -: "${AIRFLOW_HOME:="/usr/local/airflow"}" -: "${AIRFLOW__CORE__FERNET_KEY:=${FERNET_KEY:=$(python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)")}}" -: "${AIRFLOW__CORE__EXECUTOR:=${EXECUTOR:-Sequential}Executor}" - -# Load DAGs examples (default: Yes) -if [[ -z "$AIRFLOW__CORE__LOAD_EXAMPLES" && "${LOAD_EX:=n}" == n ]]; then - AIRFLOW__CORE__LOAD_EXAMPLES=False -fi - -export \ - AIRFLOW_HOME \ - AIRFLOW__CORE__EXECUTOR \ - AIRFLOW__CORE__FERNET_KEY \ - AIRFLOW__CORE__LOAD_EXAMPLES \ - -# Install custom python package if requirements.txt is present -if [ -e "/requirements.txt" ]; then - $(command -v pip) install --user -r /requirements.txt -fi - -wait_for_port() { - local name="$1" host="$2" port="$3" - local j=0 - while ! nc -z "$host" "$port" >/dev/null 2>&1 < /dev/null; do - j=$((j+1)) - if [ $j -ge $TRY_LOOP ]; then - echo >&2 "$(date) - $host:$port still not reachable, giving up" - exit 1 - fi - echo "$(date) - waiting for $name... $j/$TRY_LOOP" - sleep 5 - done -} - -# Other executors than SequentialExecutor drive the need for an SQL database, here PostgreSQL is used -if [ "$AIRFLOW__CORE__EXECUTOR" != "SequentialExecutor" ]; then - # Check if the user has provided explicit Airflow configuration concerning the database - if [ -z "$AIRFLOW__CORE__SQL_ALCHEMY_CONN" ]; then - # Default values corresponding to the default compose files - : "${POSTGRES_HOST:="postgres"}" - : "${POSTGRES_PORT:="5432"}" - : "${POSTGRES_USER:="airflow"}" - : "${POSTGRES_PASSWORD:="airflow"}" - : "${POSTGRES_DB:="airflow"}" - : "${POSTGRES_EXTRAS:-""}" - - AIRFLOW__CORE__SQL_ALCHEMY_CONN="postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}${POSTGRES_EXTRAS}" - export AIRFLOW__CORE__SQL_ALCHEMY_CONN - - # Check if the user has provided explicit Airflow configuration for the broker's connection to the database - if [ "$AIRFLOW__CORE__EXECUTOR" = "CeleryExecutor" ]; then - AIRFLOW__CELERY__RESULT_BACKEND="db+postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}${POSTGRES_EXTRAS}" - export AIRFLOW__CELERY__RESULT_BACKEND - fi - else - if [[ "$AIRFLOW__CORE__EXECUTOR" == "CeleryExecutor" && -z "$AIRFLOW__CELERY__RESULT_BACKEND" ]]; then - >&2 printf '%s\n' "FATAL: if you set AIRFLOW__CORE__SQL_ALCHEMY_CONN manually with CeleryExecutor you must also set AIRFLOW__CELERY__RESULT_BACKEND" - exit 1 - fi - - # Derive useful variables from the AIRFLOW__ variables provided explicitly by the user - POSTGRES_ENDPOINT=$(echo -n "$AIRFLOW__CORE__SQL_ALCHEMY_CONN" | cut -d '/' -f3 | sed -e 's,.*@,,') - POSTGRES_HOST=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f1) - POSTGRES_PORT=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f2) - fi - - wait_for_port "Postgres" "$POSTGRES_HOST" "$POSTGRES_PORT" -fi - -# CeleryExecutor drives the need for a Celery broker, here Redis is used -if [ "$AIRFLOW__CORE__EXECUTOR" = "CeleryExecutor" ]; then - # Check if the user has provided explicit Airflow configuration concerning the broker - if [ -z "$AIRFLOW__CELERY__BROKER_URL" ]; then - # Default values corresponding to the default compose files - : "${REDIS_PROTO:="redis://"}" - : "${REDIS_HOST:="redis"}" - : "${REDIS_PORT:="6379"}" - : "${REDIS_PASSWORD:=""}" - : "${REDIS_DBNUM:="1"}" - - # When Redis is secured by basic auth, it does not handle the username part of basic auth, only a token - if [ -n "$REDIS_PASSWORD" ]; then - REDIS_PREFIX=":${REDIS_PASSWORD}@" - else - REDIS_PREFIX= - fi - - AIRFLOW__CELERY__BROKER_URL="${REDIS_PROTO}${REDIS_PREFIX}${REDIS_HOST}:${REDIS_PORT}/${REDIS_DBNUM}" - export AIRFLOW__CELERY__BROKER_URL - else - # Derive useful variables from the AIRFLOW__ variables provided explicitly by the user - REDIS_ENDPOINT=$(echo -n "$AIRFLOW__CELERY__BROKER_URL" | cut -d '/' -f3 | sed -e 's,.*@,,') - REDIS_HOST=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f1) - REDIS_PORT=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f2) - fi - - wait_for_port "Redis" "$REDIS_HOST" "$REDIS_PORT" -fi - -case "$1" in - webserver) - airflow db init - airflow users create --username airflow --password airflow --firstname Peter --lastname Parker --role Admin --email spiderman@superhero.org - if [ "$AIRFLOW__CORE__EXECUTOR" = "LocalExecutor" ] || [ "$AIRFLOW__CORE__EXECUTOR" = "SequentialExecutor" ]; then - # With the "Local" and "Sequential" executors it should all run in one container. - airflow scheduler & - fi - exec airflow webserver - ;; - celery|scheduler) - # Give the webserver time to run db init. - sleep 10 - exec airflow "$@" - ;; - celery) - sleep 10 - exec airflow "$@" - ;; - version) - exec airflow "$@" - ;; - *) - # The command is something like bash, not an airflow subcommand. Just run it in the right environment. - exec "$@" - ;; -esac +#!/usr/bin/env bash + +# User-provided configuration must always be respected. +# +# Therefore, this script must only derives Airflow AIRFLOW__ variables from other variables +# when the user did not provide their own configuration. + +TRY_LOOP="20" + +# Global defaults and back-compat +: "${AIRFLOW_HOME:="/usr/local/airflow"}" +: "${AIRFLOW__CORE__FERNET_KEY:=${FERNET_KEY:=$(python -c "from cryptography.fernet import Fernet; FERNET_KEY = Fernet.generate_key().decode(); print(FERNET_KEY)")}}" +: "${AIRFLOW__CORE__EXECUTOR:=${EXECUTOR:-Sequential}Executor}" + +# Load DAGs examples (default: Yes) +if [[ -z "$AIRFLOW__CORE__LOAD_EXAMPLES" && "${LOAD_EX:=n}" == n ]]; then + AIRFLOW__CORE__LOAD_EXAMPLES=False +fi + +export \ + AIRFLOW_HOME \ + AIRFLOW__CORE__EXECUTOR \ + AIRFLOW__CORE__FERNET_KEY \ + AIRFLOW__CORE__LOAD_EXAMPLES \ + +# Install custom python package if requirements.txt is present +if [ -e "/requirements.txt" ]; then + $(command -v pip) install --user -r /requirements.txt +fi + +wait_for_port() { + local name="$1" host="$2" port="$3" + local j=0 + while ! nc -z "$host" "$port" >/dev/null 2>&1 < /dev/null; do + j=$((j+1)) + if [ $j -ge $TRY_LOOP ]; then + echo >&2 "$(date) - $host:$port still not reachable, giving up" + exit 1 + fi + echo "$(date) - waiting for $name... $j/$TRY_LOOP" + sleep 5 + done +} + +# Other executors than SequentialExecutor drive the need for an SQL database, here PostgreSQL is used +if [ "$AIRFLOW__CORE__EXECUTOR" != "SequentialExecutor" ]; then + # Check if the user has provided explicit Airflow configuration concerning the database + if [ -z "$AIRFLOW__CORE__SQL_ALCHEMY_CONN" ]; then + # Default values corresponding to the default compose files + : "${POSTGRES_HOST:="postgres"}" + : "${POSTGRES_PORT:="5432"}" + : "${POSTGRES_USER:="airflow"}" + : "${POSTGRES_PASSWORD:="airflow"}" + : "${POSTGRES_DB:="airflow"}" + : "${POSTGRES_EXTRAS:-""}" + + AIRFLOW__CORE__SQL_ALCHEMY_CONN="postgresql+psycopg2://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}${POSTGRES_EXTRAS}" + export AIRFLOW__CORE__SQL_ALCHEMY_CONN + + # Check if the user has provided explicit Airflow configuration for the broker's connection to the database + if [ "$AIRFLOW__CORE__EXECUTOR" = "CeleryExecutor" ]; then + AIRFLOW__CELERY__RESULT_BACKEND="db+postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB}${POSTGRES_EXTRAS}" + export AIRFLOW__CELERY__RESULT_BACKEND + fi + else + if [[ "$AIRFLOW__CORE__EXECUTOR" == "CeleryExecutor" && -z "$AIRFLOW__CELERY__RESULT_BACKEND" ]]; then + >&2 printf '%s\n' "FATAL: if you set AIRFLOW__CORE__SQL_ALCHEMY_CONN manually with CeleryExecutor you must also set AIRFLOW__CELERY__RESULT_BACKEND" + exit 1 + fi + + # Derive useful variables from the AIRFLOW__ variables provided explicitly by the user + POSTGRES_ENDPOINT=$(echo -n "$AIRFLOW__CORE__SQL_ALCHEMY_CONN" | cut -d '/' -f3 | sed -e 's,.*@,,') + POSTGRES_HOST=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f1) + POSTGRES_PORT=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f2) + fi + + wait_for_port "Postgres" "$POSTGRES_HOST" "$POSTGRES_PORT" +fi + +# CeleryExecutor drives the need for a Celery broker, here Redis is used +if [ "$AIRFLOW__CORE__EXECUTOR" = "CeleryExecutor" ]; then + # Check if the user has provided explicit Airflow configuration concerning the broker + if [ -z "$AIRFLOW__CELERY__BROKER_URL" ]; then + # Default values corresponding to the default compose files + : "${REDIS_PROTO:="redis://"}" + : "${REDIS_HOST:="redis"}" + : "${REDIS_PORT:="6379"}" + : "${REDIS_PASSWORD:=""}" + : "${REDIS_DBNUM:="1"}" + + # When Redis is secured by basic auth, it does not handle the username part of basic auth, only a token + if [ -n "$REDIS_PASSWORD" ]; then + REDIS_PREFIX=":${REDIS_PASSWORD}@" + else + REDIS_PREFIX= + fi + + AIRFLOW__CELERY__BROKER_URL="${REDIS_PROTO}${REDIS_PREFIX}${REDIS_HOST}:${REDIS_PORT}/${REDIS_DBNUM}" + export AIRFLOW__CELERY__BROKER_URL + else + # Derive useful variables from the AIRFLOW__ variables provided explicitly by the user + REDIS_ENDPOINT=$(echo -n "$AIRFLOW__CELERY__BROKER_URL" | cut -d '/' -f3 | sed -e 's,.*@,,') + REDIS_HOST=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f1) + REDIS_PORT=$(echo -n "$POSTGRES_ENDPOINT" | cut -d ':' -f2) + fi + + wait_for_port "Redis" "$REDIS_HOST" "$REDIS_PORT" +fi + +case "$1" in + webserver) + airflow db init + airflow users create --username airflow --password airflow --firstname Peter --lastname Parker --role Admin --email spiderman@superhero.org + if [ "$AIRFLOW__CORE__EXECUTOR" = "LocalExecutor" ] || [ "$AIRFLOW__CORE__EXECUTOR" = "SequentialExecutor" ]; then + # With the "Local" and "Sequential" executors it should all run in one container. + airflow scheduler & + fi + exec airflow webserver + ;; + celery|scheduler) + # Give the webserver time to run db init. + sleep 10 + exec airflow "$@" + ;; + celery) + sleep 10 + exec airflow "$@" + ;; + version) + exec airflow "$@" + ;; + *) + # The command is something like bash, not an airflow subcommand. Just run it in the right environment. + exec "$@" + ;; +esac From 9c720d9ded7db9857ff18dc66aa974e0be28c35f Mon Sep 17 00:00:00 2001 From: Higor Carneiro Passos Date: Tue, 30 Nov 2021 16:37:33 -0300 Subject: [PATCH 20/20] added some deploy parameters to worker container --- .gitignore | 1 + docker-compose-CeleryExecutor.yml | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/.gitignore b/.gitignore index c4fc2162..9b411a5a 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ __pycache__ microdados_enade_2019* igti_bootcamp* + treino02.py treino03.py treino04.py diff --git a/docker-compose-CeleryExecutor.yml b/docker-compose-CeleryExecutor.yml index 5edefc97..8a0cfe06 100644 --- a/docker-compose-CeleryExecutor.yml +++ b/docker-compose-CeleryExecutor.yml @@ -96,3 +96,11 @@ services: # - POSTGRES_DB=airflow # - REDIS_PASSWORD=redispass command: celery worker + deploy: + resources: + limits: + cpus: '0.01' + memory: 1024M + reservations: + cpus: '0.001' + memory: 256M \ No newline at end of file