[AIRFLOW-6817] Lazy-load airflow.DAG to keep user-facing API untouc…

…hed (#7517)

`from airflow import DAG` is _such_ a common pattern that it's worth
making sure this stays working.

Since we are now on Python 3 we can use PEP-562 to have officially
supported lazy-loading of attributes on a module.

Since the example_dags will be referred to/copied by users I have
updated them all back to import from the airflow module, but have left
any internal uses untouched.

UPDATING.md

@@ -63,7 +63,7 @@ https://developers.google.com/style/inclusive-documentation
 
 ### Removed sub-package imports from `airflow/__init__.py`
 
-The imports `LoggingMixin`, `conf`, `AirflowException`, and `DAG` have been removed from `airflow/__init__.py`.
+The imports `LoggingMixin`, `conf`, and `AirflowException` have been removed from `airflow/__init__.py`.
 All implicit references of these objects will no longer be valid. To migrate, all usages of each old path must be
 replaced with its corresponding new path.
 
@@ -72,7 +72,6 @@ replaced with its corresponding new path.
 | ``airflow.LoggingMixin``     | ``airflow.utils.log.logging_mixin.LoggingMixin`` |
 | ``airflow.conf``             | ``airflow.configuration.conf``                   |
 | ``airflow.AirflowException`` | ``airflow.exceptions.AirflowException``          |
-| ``airflow.DAG``              | ``airflow.models.dag.DAG``                  |
 
 ### Success Callback will be called when a task in marked as success from UI
 

airflow/__init__.py

@@ -28,17 +28,46 @@
 
 # flake8: noqa: F401
 # pylint: disable=wrong-import-position
+import sys
 from typing import Callable, Optional
 
 from airflow import settings
 from airflow import version
 
 __version__ = version.version
 
+__all__ = ['__version__', 'login', 'DAG']
+
 settings.initialize()
 
 from airflow.plugins_manager import integrate_plugins
 
 login: Optional[Callable] = None
 
 integrate_plugins()
+
+
+PY37 = sys.version_info >= (3, 7)
+
+
+def __getattr__(name):
+    # PEP-562: Lazy loaded attributes on python modules
+    if name == "DAG":
+        from airflow.models.dag import DAG # pylint: disable=redefined-outer-name
+        return DAG
+    raise AttributeError(f"module {__name__} has no attribute {name}")
+
+
+# This is never executed, but tricks static analyzers (PyDev, PyCharm,
+# pylint, etc.) into knowing the types of these symbols, and what
+# they contain.
+STATICA_HACK = True
+globals()['kcah_acitats'[::-1].upper()] = False
+if STATICA_HACK:  # pragma: no cover
+    from airflow.models.dag import DAG
+
+
+if not PY37:
+    from pep562 import Pep562
+
+    Pep562(__name__)

airflow/example_dags/example_bash_operator.py

@@ -20,7 +20,7 @@
 
 from datetime import timedelta
 
-from airflow.models import DAG
+from airflow import DAG
 from airflow.operators.bash import BashOperator
 from airflow.operators.dummy_operator import DummyOperator
 from airflow.utils.dates import days_ago

airflow/example_dags/example_branch_operator.py

@@ -20,7 +20,7 @@
 
 import random
 
-from airflow.models import DAG
+from airflow import DAG
 from airflow.operators.dummy_operator import DummyOperator
 from airflow.operators.python import BranchPythonOperator
 from airflow.utils.dates import days_ago

airflow/example_dags/example_branch_python_dop_operator_3.py

@@ -21,7 +21,7 @@
 or skipped on alternating runs.
 """
 
-from airflow.models import DAG
+from airflow import DAG
 from airflow.operators.dummy_operator import DummyOperator
 from airflow.operators.python import BranchPythonOperator
 from airflow.utils.dates import days_ago

airflow/example_dags/example_external_task_marker_dag.py

@@ -28,7 +28,7 @@
 
 import datetime
 
-from airflow.models import DAG
+from airflow import DAG
 from airflow.operators.dummy_operator import DummyOperator
 from airflow.sensors.external_task_sensor import ExternalTaskMarker, ExternalTaskSensor
 

airflow/example_dags/example_kubernetes_executor.py

@@ -20,8 +20,8 @@
 """
 import os
 
+from airflow import DAG
 from airflow.example_dags.libs.helper import print_stuff
-from airflow.models import DAG
 from airflow.operators.python import PythonOperator
 from airflow.utils.dates import days_ago
 

airflow/example_dags/example_kubernetes_executor_config.py

@@ -20,8 +20,8 @@
 """
 import os
 
+from airflow import DAG
 from airflow.example_dags.libs.helper import print_stuff
-from airflow.models import DAG
 from airflow.operators.python import PythonOperator
 from airflow.utils.dates import days_ago
 

airflow/example_dags/example_latest_only.py

@@ -20,7 +20,7 @@
 
 import datetime as dt
 
-from airflow.models import DAG
+from airflow import DAG
 from airflow.operators.dummy_operator import DummyOperator
 from airflow.operators.latest_only_operator import LatestOnlyOperator
 from airflow.utils.dates import days_ago

airflow/example_dags/example_latest_only_with_trigger.py

@@ -20,7 +20,7 @@
 """
 import datetime as dt
 
-from airflow.models import DAG
+from airflow import DAG
 from airflow.operators.dummy_operator import DummyOperator
 from airflow.operators.latest_only_operator import LatestOnlyOperator
 from airflow.utils.dates import days_ago

airflow/example_dags/example_passing_params_via_test_command.py

@@ -20,7 +20,7 @@
 
 from datetime import timedelta
 
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.operators.bash import BashOperator
 from airflow.operators.python import PythonOperator
 from airflow.utils.dates import days_ago

airflow/example_dags/example_python_operator.py

@@ -21,7 +21,7 @@
 import time
 from pprint import pprint
 
-from airflow.models import DAG
+from airflow import DAG
 from airflow.operators.python import PythonOperator, PythonVirtualenvOperator
 from airflow.utils.dates import days_ago
 

airflow/example_dags/example_short_circuit_operator.py

@@ -17,7 +17,7 @@
 # under the License.
 
 """Example DAG demonstrating the usage of the ShortCircuitOperator."""
-from airflow.models import DAG
+from airflow import DAG
 from airflow.models.baseoperator import chain
 from airflow.operators.dummy_operator import DummyOperator
 from airflow.operators.python import ShortCircuitOperator

airflow/example_dags/example_skip_dag.py

@@ -18,8 +18,8 @@
 
 """Example DAG demonstrating the DummyOperator and a custom DummySkipOperator which skips by default."""
 
+from airflow import DAG
 from airflow.exceptions import AirflowSkipException
-from airflow.models import DAG
 from airflow.operators.dummy_operator import DummyOperator
 from airflow.utils.dates import days_ago
 

airflow/example_dags/example_subdag_operator.py

@@ -18,8 +18,8 @@
 
 """Example DAG demonstrating the usage of the SubDagOperator."""
 
+from airflow import DAG
 from airflow.example_dags.subdags.subdag import subdag
-from airflow.models import DAG
 from airflow.operators.dummy_operator import DummyOperator
 from airflow.operators.subdag_operator import SubDagOperator
 from airflow.utils.dates import days_ago

airflow/example_dags/example_trigger_controller_dag.py

@@ -21,7 +21,7 @@
 1. 1st DAG (example_trigger_controller_dag) holds a TriggerDagRunOperator, which will trigger the 2nd DAG
 2. 2nd DAG (example_trigger_target_dag) which will be triggered by the TriggerDagRunOperator in the 1st DAG
 """
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.operators.dagrun_operator import TriggerDagRunOperator
 from airflow.utils.dates import days_ago
 

airflow/example_dags/example_trigger_target_dag.py

@@ -22,7 +22,7 @@
 2. 2nd DAG (example_trigger_target_dag) which will be triggered by the TriggerDagRunOperator in the 1st DAG
 """
 
-from airflow.models import DAG
+from airflow import DAG
 from airflow.operators.bash import BashOperator
 from airflow.operators.python import PythonOperator
 from airflow.utils.dates import days_ago

airflow/example_dags/example_xcom.py

@@ -17,7 +17,7 @@
 # under the License.
 
 """Example DAG demonstrating the usage of XComs."""
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.operators.python import PythonOperator
 from airflow.utils.dates import days_ago
 

airflow/example_dags/subdags/subdag.py

@@ -18,7 +18,7 @@
 
 """Helper function to generate a DAG and operators given some arguments."""
 
-from airflow.models import DAG
+from airflow import DAG
 from airflow.operators.dummy_operator import DummyOperator
 
 

airflow/example_dags/test_utils.py

@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 """Used for unit tests"""
-from airflow.models import DAG
+from airflow import DAG
 from airflow.operators.bash import BashOperator
 from airflow.utils.dates import days_ago
 

airflow/example_dags/tutorial.py

@@ -26,7 +26,7 @@
 
 # [START import_module]
 # The DAG object; we'll need this to instantiate a DAG
-from airflow.models.dag import DAG
+from airflow import DAG
 # Operators; we need this to operate!
 from airflow.operators.bash import BashOperator
 from airflow.utils.dates import days_ago

airflow/providers/amazon/aws/example_dags/example_emr_job_flow_automatic_steps.py

@@ -20,7 +20,7 @@
 """
 from datetime import timedelta
 
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.providers.amazon.aws.operators.emr_create_job_flow import EmrCreateJobFlowOperator
 from airflow.providers.amazon.aws.sensors.emr_job_flow import EmrJobFlowSensor
 from airflow.utils.dates import days_ago

airflow/providers/amazon/aws/example_dags/example_emr_job_flow_manual_steps.py

@@ -23,7 +23,7 @@
 """
 from datetime import timedelta
 
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.providers.amazon.aws.operators.emr_add_steps import EmrAddStepsOperator
 from airflow.providers.amazon.aws.operators.emr_create_job_flow import EmrCreateJobFlowOperator
 from airflow.providers.amazon.aws.operators.emr_terminate_job_flow import EmrTerminateJobFlowOperator

airflow/providers/apache/hive/example_dags/example_twitter_dag.py

@@ -30,7 +30,7 @@
 """
 from datetime import date, timedelta
 
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.operators.bash import BashOperator
 from airflow.operators.python import PythonOperator
 from airflow.providers.apache.hive.operators.hive import HiveOperator

airflow/providers/apache/livy/example_dags/example_livy.py

@@ -21,7 +21,7 @@
 using the Java and Python executables provided in the example library.
 """
 
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.providers.apache.livy.operators.livy import LivyOperator
 from airflow.utils.dates import days_ago
 

airflow/providers/apache/pig/example_dags/example_pig.py

@@ -18,7 +18,7 @@
 
 """Example DAG demonstrating the usage of the PigOperator."""
 
-from airflow.models import DAG
+from airflow import DAG
 from airflow.providers.apache.pig.operators.pig import PigOperator
 from airflow.utils.dates import days_ago
 

airflow/providers/cncf/kubernetes/example_dags/example_kubernetes.py

@@ -20,7 +20,7 @@
 """
 import logging
 
-from airflow.models import DAG
+from airflow import DAG
 from airflow.utils.dates import days_ago
 
 log = logging.getLogger(__name__)

airflow/providers/databricks/example_dags/example_databricks.py

@@ -31,7 +31,7 @@
 https://docs.databricks.com/api/latest/jobs.html#runstate
 """
 
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.providers.databricks.operators.databricks import DatabricksSubmitRunOperator
 from airflow.utils.dates import days_ago
 

airflow/providers/dingding/example_dags/example_dingding.py

@@ -20,7 +20,7 @@
 """
 from datetime import timedelta
 
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.providers.dingding.operators.dingding import DingdingOperator
 from airflow.utils.dates import days_ago
 

airflow/providers/docker/example_dags/example_docker.py

@@ -17,7 +17,7 @@
 # under the License.
 from datetime import timedelta
 
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.operators.bash import BashOperator
 from airflow.providers.docker.operators.docker import DockerOperator
 from airflow.utils.dates import days_ago

airflow/providers/docker/example_dags/example_docker_copy_data.py

@@ -27,7 +27,7 @@
 
 from datetime import timedelta
 
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.operators.bash import BashOperator
 from airflow.operators.python import ShortCircuitOperator
 from airflow.providers.docker.operators.docker import DockerOperator

airflow/providers/docker/example_dags/example_docker_swarm.py

@@ -17,7 +17,7 @@
 # under the License.
 from datetime import timedelta
 
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.providers.docker.operators.docker_swarm import DockerSwarmOperator
 from airflow.utils.dates import days_ago
 

airflow/providers/google/cloud/example_dags/example_dlp.py

@@ -28,7 +28,7 @@
 
 from google.cloud.dlp_v2.types import ContentItem, InspectConfig, InspectTemplate
 
-from airflow.models import DAG
+from airflow import DAG
 from airflow.providers.google.cloud.operators.dlp import (
     CloudDLPCreateInspectTemplateOperator, CloudDLPDeleteInspectTemplateOperator,
     CloudDLPInspectContentOperator,

airflow/providers/google/cloud/example_dags/example_tasks.py

@@ -29,7 +29,7 @@
 from google.cloud.tasks_v2.types import Queue
 from google.protobuf import timestamp_pb2
 
-from airflow.models import DAG
+from airflow import DAG
 from airflow.providers.google.cloud.operators.tasks import (
     CloudTasksQueueCreateOperator, CloudTasksTaskCreateOperator, CloudTasksTaskRunOperator,
 )

airflow/providers/http/example_dags/example_http.py

@@ -21,7 +21,7 @@
 import json
 from datetime import timedelta
 
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.providers.http.operators.http import SimpleHttpOperator
 from airflow.providers.http.sensors.http import HttpSensor
 from airflow.utils.dates import days_ago

airflow/providers/jenkins/example_dags/example_jenkins_job_trigger.py

@@ -19,7 +19,7 @@
 
 from six.moves.urllib.request import Request
 
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.operators.python import PythonOperator
 from airflow.providers.jenkins.hooks.jenkins import JenkinsHook
 from airflow.providers.jenkins.operators.jenkins_job_trigger import JenkinsJobTriggerOperator

airflow/providers/microsoft/azure/example_dags/example_azure_container_instances.py

@@ -20,7 +20,7 @@
 """
 from datetime import datetime, timedelta
 
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.providers.microsoft.azure.operators.azure_container_instances import (
     AzureContainerInstancesOperator,
 )

airflow/providers/microsoft/azure/example_dags/example_azure_cosmosdb.py

@@ -26,7 +26,7 @@
 this example.*
 """
 
-from airflow.models.dag import DAG
+from airflow import DAG
 from airflow.providers.microsoft.azure.operators.azure_cosmos import AzureCosmosInsertDocumentOperator
 from airflow.providers.microsoft.azure.sensors.azure_cosmos import AzureCosmosDocumentSensor
 from airflow.utils import dates