Change regular expression to exclude double quote and newline (#39991)

This change tackles a change in Cloud Composer's default storage mechanism for logs as documented in the release notes on [March 26, 2024](https://cloud.google.com/composer/docs/release-notes#March_26_2024) and [May 11, 2024](https://cloud.google.com/composer/docs/release-notes#May_11_2024). This causes issue for Beam operator which uses regular expression to identify the Dataflow Job ID from the logs.

Co-authored-by: Yuan Chuan Kee <[email protected]>

airflow/providers/google/cloud/hooks/dataflow.py

@@ -71,7 +71,7 @@
 
 
 JOB_ID_PATTERN = re.compile(
-    r"Submitted job: (?P<job_id_java>.*)|Created job with id: \[(?P<job_id_python>.*)\]"
+    r"Submitted job: (?P<job_id_java>[^\"\n]*)|Created job with id: \[(?P<job_id_python>[^\"\n]*)\]"
 )
 
 T = TypeVar("T", bound=Callable)

tests/providers/google/cloud/hooks/test_dataflow.py

@@ -1877,6 +1877,19 @@ def test_fetch_job_autoscaling_events_by_id(self, mock_fetch_responses):
 > gcloud dataflow jobs --project=XXX cancel --region=europe-west3 {TEST_JOB_ID}
 """
 
+CLOUD_COMPOSER_CLOUD_LOGGING_APACHE_BEAM_V_2_56_0_JAVA_SDK_LOG = f"""\
+WARNING - {{"message":"org.apache.beam.runners.dataflow.DataflowRunner - Dataflow SDK version: 2.56.0",\
+"severity":"INFO"}}
+WARNING - {{"message":"org.apache.beam.runners.dataflow.DataflowRunner - To access the Dataflow monitoring\\
+console, please navigate to https://console.cloud.google.com/dataflow/jobs/europe-west3/{TEST_JOB_ID}?project
+\u003dXXXX","severity":"INFO"}}
+WARNING - {{"message":"org.apache.beam.runners.dataflow.DataflowRunner - Submitted job: {TEST_JOB_ID}",\
+"severity":"INFO"}}
+WARNING - {{"message":"org.apache.beam.runners.dataflow.DataflowRunner - To cancel the job using the \
+\u0027gcloud\u0027 tool, run:\n\u003e gcloud dataflow jobs --project\u003dXXX cancel --region\u003deurope-\
+west3 {TEST_JOB_ID}","severity":"INFO"}}
+"""
+
 APACHE_BEAM_V_2_14_0_PYTHON_SDK_LOG = f""""\
 INFO:root:Completed GCS upload to gs://test-dataflow-example/staging/start-python-job-local-5bcf3d71.\
 1592286375.000962/apache_beam-2.14.0-cp37-cp37m-manylinux1_x86_64.whl in 0 seconds.
@@ -1926,6 +1939,10 @@ class TestDataflow:
         [
             pytest.param(APACHE_BEAM_V_2_14_0_JAVA_SDK_LOG, id="apache-beam-2.14.0-JDK"),
             pytest.param(APACHE_BEAM_V_2_22_0_JAVA_SDK_LOG, id="apache-beam-2.22.0-JDK"),
+            pytest.param(
+                CLOUD_COMPOSER_CLOUD_LOGGING_APACHE_BEAM_V_2_56_0_JAVA_SDK_LOG,
+                id="cloud-composer-cloud-logging-apache-beam-2.56.0-JDK",
+            ),
             pytest.param(APACHE_BEAM_V_2_14_0_PYTHON_SDK_LOG, id="apache-beam-2.14.0-Python"),
             pytest.param(APACHE_BEAM_V_2_22_0_PYTHON_SDK_LOG, id="apache-beam-2.22.0-Python"),
         ],