Upgrade to support Google Ads v10 (#22965)

Dockerfile

@@ -1342,8 +1342,7 @@ ARG INSTALL_FROM_PYPI="true"
 # Force them on the main Airflow package.
 # * certifi<2021.0.0 required to keep snowflake happy
 # * dill<0.3.3 required by apache-beam
-# * google-ads<14.0.1 required to prevent updating google-python-api>=2.0.0
-ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 certifi<2021.0.0 google-ads<14.0.1"
+ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 certifi<2021.0.0"
 
 ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \
     INSTALL_FROM_DOCKER_CONTEXT_FILES=${INSTALL_FROM_DOCKER_CONTEXT_FILES} \

Dockerfile.ci

@@ -1588,8 +1588,7 @@ RUN echo "Airflow version: ${AIRFLOW_VERSION}"
 # force them on the main Airflow package. Those limitations are:
 # * certifi<2021.0.0: required by snowflake provider
 # * dill<0.3.3 required by apache-beam
-# * google-ads<14.0.1 required to prevent updating google-python-api>=2.0.0
-ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 certifi<2021.0.0 google-ads<14.0.1"
+ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 certifi<2021.0.0"
 ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
 ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \
     UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES}

airflow/providers/google/CHANGELOG.rst

@@ -18,6 +18,22 @@
 Changelog
 ---------
 
+7.0.0
+.....
+
+Breaking changes
+~~~~~~~~~~~~~~~~
+
+* ``apache-airflow-providers-google uses deprecated Google Ads API V8 (#22111)``
+
+.. warning:: The underlying google-ads library has been updated
+
+   This drops support for versions v6 and v7 of the Google Ads API, and updates
+   the default version of the Google Ads API from the deprecated v8 to v10.
+
+   For more information, see `Deprecation and sunset <https://developers.google.com/google-ads/api/docs/sunset-dates>`_
+   and `Upgrading to the newest version <https://developers.google.com/google-ads/api/docs/version-migration>`_
+
 6.8.0
 .....
 

airflow/providers/google/ads/hooks/ads.py

@@ -27,10 +27,11 @@
 
 from google.ads.googleads.client import GoogleAdsClient
 from google.ads.googleads.errors import GoogleAdsException
-from google.ads.googleads.v8.services.types.google_ads_service import GoogleAdsRow
+from google.ads.googleads.v10.services.services.customer_service import CustomerServiceClient
+from google.ads.googleads.v10.services.services.google_ads_service import GoogleAdsServiceClient
+from google.ads.googleads.v10.services.types.google_ads_service import GoogleAdsRow, SearchGoogleAdsRequest
 from google.api_core.page_iterator import GRPCIterator
 from google.auth.exceptions import GoogleAuthError
-from googleapiclient.discovery import Resource
 
 from airflow import AirflowException
 from airflow.hooks.base import BaseHook
@@ -51,12 +52,12 @@ class GoogleAdsHook(BaseHook):
             {
                 "google_ads_client": {
                     "developer_token": "{{ INSERT_TOKEN }}",
-                    "path_to_private_key_file": null,
-                    "delegated_account": "{{ INSERT_DELEGATED_ACCOUNT }}"
+                    "json_key_file_path": null,
+                    "impersonated_email": "{{ INSERT_IMPERSONATED_EMAIL }}"
                 }
             }
 
-        The ``path_to_private_key_file`` is resolved by the hook using credentials from gcp_conn_id.
+        The ``json_key_file_path`` is resolved by the hook using credentials from gcp_conn_id.
         https://developers.google.com/google-ads/api/docs/client-libs/python/oauth-service
 
     .. seealso::
@@ -75,7 +76,7 @@ class GoogleAdsHook(BaseHook):
     :rtype: list[GoogleAdsRow]
     """
 
-    default_api_version = "v8"
+    default_api_version = "v10"
 
     def __init__(
         self,
@@ -155,13 +156,13 @@ def list_accessible_customers(self) -> List[str]:
             raise
 
     @cached_property
-    def _get_service(self) -> Resource:
+    def _get_service(self) -> GoogleAdsServiceClient:
         """Connects and authenticates with the Google Ads API using a service account"""
         client = self._get_client
         return client.get_service("GoogleAdsService", version=self.api_version)
 
     @cached_property
-    def _get_client(self) -> Resource:
+    def _get_client(self) -> GoogleAdsClient:
         with NamedTemporaryFile("w", suffix=".json") as secrets_temp:
             self._get_config()
             self._update_config_with_secret(secrets_temp)
@@ -173,7 +174,7 @@ def _get_client(self) -> Resource:
                 raise
 
     @cached_property
-    def _get_customer_service(self) -> Resource:
+    def _get_customer_service(self) -> CustomerServiceClient:
         """Connects and authenticates with the Google Ads API using a service account"""
         with NamedTemporaryFile("w", suffix=".json") as secrets_temp:
             self._get_config()
@@ -207,7 +208,7 @@ def _update_config_with_secret(self, secrets_temp: IO[str]) -> None:
         secrets_temp.write(secret)
         secrets_temp.flush()
 
-        self.google_ads_config["path_to_private_key_file"] = secrets_temp.name
+        self.google_ads_config["json_key_file_path"] = secrets_temp.name
 
     def _search(
         self, client_ids: List[str], query: str, page_size: int = 10000, **kwargs
@@ -226,7 +227,7 @@ def _search(
 
         iterators = []
         for client_id in client_ids:
-            request = self._get_client.get_type("SearchGoogleAdsRequest")
+            request = self._get_client.get_type("SearchGoogleAdsRequest")  # type: SearchGoogleAdsRequest
             request.customer_id = client_id
             request.query = query
             request.page_size = page_size

docs/apache-airflow-providers-google/index.rst

@@ -91,7 +91,7 @@ PIP package                             Version required
 ======================================  ====================
 ``apache-airflow``                      ``>=2.1.0``
 ``PyOpenSSL``
-``google-ads``                          ``>=12.0.0,<14.0.1``
+``google-ads``                          ``>=15.1.1``
 ``google-api-core``                     ``>=1.25.1,<3.0.0``
 ``google-api-python-client``            ``>=1.6.0,<2.0.0``
 ``google-auth-httplib2``                ``>=0.0.1``

setup.cfg

@@ -91,7 +91,9 @@ install_requires =
     # Cattrs upgrades were known to break lineage https://github.com/apache/airflow/issues/16172
     # TODO: Cattrs is now at 3.8 version so we should attempt to upgrade cattrs soon.
     cattrs~=1.1, !=1.7.*
-    colorlog>=4.0.2
+    # Colorlog 6.x merges TTYColoredFormatter into ColoredFormatter, breaking backwards compatibility with 4.x
+    # Update CustomTTYColoredFormatter to remove
+    colorlog>=4.0.2, <5.0
     connexion[swagger-ui,flask]>=2.10.0
     cron-descriptor>=1.2.24
     croniter>=0.3.17

setup.py

@@ -324,10 +324,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     # Introduced breaking changes across the board. Those libraries should be upgraded soon
     # TODO: Upgrade all Google libraries that are limited to <2.0.0
     'PyOpenSSL',
-    # The Google Ads 14.0.1 breaks PIP and eager upgrade as it requires
-    # google-api-core>=2.0.0 which cannot be used yet (see below comment)
-    # and https://github.com/apache/airflow/issues/18705#issuecomment-933746150
-    'google-ads>=12.0.0,<14.0.1',
+    'google-ads>=15.1.1',
     'google-api-core>=2.7.0,<3.0.0',
     'google-api-python-client>=1.6.0,<2.0.0',
     'google-auth>=1.0.0',

tests/providers/google/ads/operators/test_ads.py

@@ -37,7 +37,7 @@
 
 gcp_conn_id = "gcp_conn_id"
 google_ads_conn_id = "google_ads_conn_id"
-api_version = "v8"
+api_version = "v10"
 
 
 class TestGoogleAdsListAccountsOperator: