Skip to content

Commit

Permalink
add separate example dags and system tests for GCSToGoogleSheetsOpera…
Browse files Browse the repository at this point in the history
…tor (#9066)

* add separate example dag and system test for GCSToGoogleSheetsOperator

* remove gcs_to_sheets from missing example dags

* fix doc error
  • Loading branch information
ephraimbuddy committed May 29, 2020
1 parent 5cf46fa commit a779c4d
Show file tree
Hide file tree
Showing 4 changed files with 106 additions and 6 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

import os

from airflow import models
from airflow.providers.google.cloud.operators.sheets_to_gcs import GoogleSheetsToGCSOperator
from airflow.providers.google.suite.operators.gcs_to_sheets import GCSToGoogleSheetsOperator
from airflow.utils.dates import days_ago

BUCKET = os.environ.get("GCP_GCS_BUCKET", "example-test-bucket3")
SPREADSHEET_ID = os.environ.get("SPREADSHEET_ID", "example-spreadsheetID")
NEW_SPREADSHEET_ID = os.environ.get("NEW_SPREADSHEET_ID", "1234567890qwerty")

default_args = {"start_date": days_ago(1)}

with models.DAG(
"example_gcs_to_sheets",
default_args=default_args,
schedule_interval=None, # Override to match your needs
tags=["example"],
) as dag:

upload_sheet_to_gcs = GoogleSheetsToGCSOperator(
task_id="upload_sheet_to_gcs",
destination_bucket=BUCKET,
spreadsheet_id=SPREADSHEET_ID,
)

# [START upload_gcs_to_sheets]
upload_gcs_to_sheet = GCSToGoogleSheetsOperator(
task_id="upload_gcs_to_sheet",
bucket_name=BUCKET,
object_name="{{ task_instance.xcom_pull('upload_sheet_to_gcs')[0] }}",
spreadsheet_id=NEW_SPREADSHEET_ID,
)
# [END upload_gcs_to_sheets]

upload_sheet_to_gcs >> upload_gcs_to_sheet
10 changes: 5 additions & 5 deletions docs/howto/operator/gcp/gcs_to_sheets.rst
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,13 @@ Upload data from GCS to Google Sheets
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

To upload data from Google Cloud Storage to Google Spreadsheet you can use the
:class:`~airflow.providers.google.cloud.operators.sheets_to_gcs.GoogleSheetsToGCSOperator`.
:class:`~airflow.providers.google.suite.operators.gcs_to_sheets.GCSToGoogleSheetsOperator`.

.. exampleinclude:: ../../../../airflow/providers/google/suite/example_dags/example_sheets.py
.. exampleinclude:: ../../../../airflow/providers/google/suite/example_dags/example_gcs_to_sheets.py
:language: python
:dedent: 4
:start-after: [START upload_sheet_to_gcs]
:end-before: [END upload_sheet_to_gcs]
:start-after: [START upload_gcs_to_sheets]
:end-before: [END upload_gcs_to_sheets]

You can use :ref:`Jinja templating <jinja-templating>` with
:template-fields:`airflow.providers.google.cloud.operators.sheets_to_gcs.GoogleSheetsToGCSOperator`.
:template-fields:`airflow.providers.google.suite.operators.gcs_to_sheets.GCSToGoogleSheetsOperator`.
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

import pytest

from airflow.providers.google.suite.example_dags.example_gcs_to_sheets import BUCKET
from tests.providers.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY
from tests.test_utils.gcp_system_helpers import GSUITE_DAG_FOLDER, GoogleSystemTest, provide_gcp_context

# Required scopes
SCOPES = [
'https://www.googleapis.com/auth/drive',
'https://www.googleapis.com/auth/cloud-platform',
]


@pytest.mark.backend("mysql", "postgres")
@pytest.mark.credential_file(GCP_GCS_KEY)
class GoogleSheetsToGCSExampleDagsSystemTest(GoogleSystemTest):

@provide_gcp_context(GCP_GCS_KEY)
def setUp(self):
super().setUp()
self.create_gcs_bucket(BUCKET)

@provide_gcp_context(GCP_GCS_KEY, scopes=SCOPES)
def test_run_example_dag_function(self):
self.run_dag('example_gcs_to_sheets', GSUITE_DAG_FOLDER)

@provide_gcp_context(GCP_GCS_KEY)
def tearDown(self):
self.delete_gcs_bucket(BUCKET)
super().tearDown()
1 change: 0 additions & 1 deletion tests/test_project_structure.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,6 @@ class TestGoogleProviderProjectStructure(unittest.TestCase):
('cloud', 'mysql_to_gcs'),
('cloud', 'mssql_to_gcs'),
('cloud', 'local_to_gcs'),
('suite', 'gcs_to_sheets'),
}

MISSING_DOC_GUIDES = {
Expand Down

0 comments on commit a779c4d

Please sign in to comment.