Skip to content

Commit 078bfaf

Browse files
ephraimbuddyKamil Breguła
ephraimbuddy
and
Kamil Breguła
authored
Extract missing gcs_to_local example DAG from gcs example (#10767)
Co-authored-by: Kamil Breguła <kamil.bregula@polidea.com>
1 parent 3c3342f commit 078bfaf

4 files changed

Lines changed: 88 additions & 2 deletions

File tree

‎airflow/providers/google/cloud/example_dags/example_gcs_to_local.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
import os
19+
20+
from airflow import models
21+
from airflow.providers.google.cloud.transfers.gcs_to_local import GCSToLocalFilesystemOperator
22+
from airflow.utils.dates import days_ago
23+
24+
PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-id")
25+
BUCKET = os.environ.get("GCP_GCS_BUCKET", "test-gcs-example-bucket")
26+
27+
PATH_TO_REMOTE_FILE = os.environ.get("GCP_GCS_PATH_TO_UPLOAD_FILE", "test-gcs-example-remote.txt")
28+
PATH_TO_LOCAL_FILE = os.environ.get("GCP_GCS_PATH_TO_SAVED_FILE", "test-gcs-example-local.txt")
29+
30+
with models.DAG(
31+
"example_gcs_to_local", start_date=days_ago(1), schedule_interval=None, tags=['example'],
32+
) as dag:
33+
# [START howto_operator_gcs_download_file_task]
34+
download_file = GCSToLocalFilesystemOperator(
35+
task_id="download_file", object_name=PATH_TO_REMOTE_FILE, bucket=BUCKET, filename=PATH_TO_LOCAL_FILE,
36+
)
37+
# [END howto_operator_gcs_download_file_task]

‎docs/howto/operator/google/transfer/gcs_to_local.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ data from GCS to local filesystem.
4242

4343
Below is an example of using this operator to upload a file to GCS.
4444

45-
.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_gcs.py
45+
.. exampleinclude:: /../airflow/providers/google/cloud/example_dags/example_gcs_to_local.py
4646
:language: python
4747
:dedent: 0
4848
:start-after: [START howto_operator_gcs_download_file_task]

‎tests/providers/google/cloud/transfers/test_gcs_to_local_system.py‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
import os
19+
20+
import pytest
21+
22+
from airflow.providers.google.cloud.example_dags.example_gcs_to_local import (
23+
BUCKET,
24+
PATH_TO_REMOTE_FILE,
25+
PATH_TO_LOCAL_FILE,
26+
)
27+
from tests.providers.google.cloud.utils.gcp_authenticator import GCP_GCS_KEY
28+
from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context
29+
30+
31+
@pytest.mark.backend("mysql", "postgres")
32+
@pytest.mark.credential_file(GCP_GCS_KEY)
33+
class GoogleCloudStorageExampleDagsTest(GoogleSystemTest):
34+
@provide_gcp_context(GCP_GCS_KEY)
35+
def setUp(self):
36+
super().setUp()
37+
self.create_gcs_bucket(BUCKET)
38+
self.upload_content_to_gcs(
39+
lines=f"{os.urandom(1 * 1024 * 1024)}", bucket=BUCKET, filename=PATH_TO_REMOTE_FILE
40+
)
41+
42+
@provide_gcp_context(GCP_GCS_KEY)
43+
def tearDown(self):
44+
self.delete_gcs_bucket(BUCKET)
45+
os.remove(PATH_TO_LOCAL_FILE)
46+
super().tearDown()
47+
48+
@provide_gcp_context(GCP_GCS_KEY)
49+
def test_run_example_dag(self):
50+
self.run_dag('example_gcs_to_local', CLOUD_DAG_FOLDER)

‎tests/test_project_structure.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,6 @@ class TestGoogleProviderProjectStructure(unittest.TestCase):
115115
('cloud', 'cassandra_to_gcs'),
116116
('cloud', 'mysql_to_gcs'),
117117
('cloud', 'mssql_to_gcs'),
118-
('cloud', 'gcs_to_local'),
119118
('ads', 'ads_to_gcs'),
120119
}
121120

0 commit comments

Comments
 (0)