Skip to content

Commit 4e09c64

Browse files
potiukpotiuk
authored
Adds GCP Secret Manager Hook (#9368)
* Adds GCP Secret Manager Hook
1 parent 880b65a commit 4e09c64

17 files changed

Lines changed: 597 additions & 81 deletions

‎airflow/contrib/secrets/gcp_secrets_manager.py‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,30 @@
1616
# specific language governing permissions and limitations
1717
# under the License.
1818

19-
"""This module is deprecated. Please use `airflow.providers.google.cloud.secrets.secrets_manager`."""
19+
"""This module is deprecated. Please use `airflow.providers.google.cloud.secrets.secret_manager`."""
2020

2121
import warnings
2222

2323
# pylint: disable=unused-import
24-
from airflow.providers.google.cloud.secrets.secrets_manager import CloudSecretsManagerBackend # noqa
24+
from airflow.providers.google.cloud.secrets.secret_manager import CloudSecretManagerBackend
2525

2626
warnings.warn(
27-
"This module is deprecated. Please use `airflow.providers.google.cloud.secrets.secrets_manager`.",
27+
"This module is deprecated. Please use `airflow.providers.google.cloud.secrets.secret_manager`.",
2828
DeprecationWarning,
2929
stacklevel=2,
3030
)
31+
32+
33+
class CloudSecretsManagerBackend(CloudSecretManagerBackend):
34+
"""
35+
This class is deprecated.
36+
Please use `airflow.providers.google.cloud.secrets.secret_manager.CloudSecretManagerBackend`.
37+
"""
38+
39+
def __init__(self, *args, **kwargs):
40+
warnings.warn(
41+
"""This class is deprecated.
42+
Please use `airflow.providers.google.cloud.secrets.secret_manager.CloudSecretManagerBackend`.""",
43+
DeprecationWarning, stacklevel=2
44+
)
45+
super().__init__(*args, **kwargs)

‎airflow/providers/google/cloud/_internal_client/__init__.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.

‎airflow/providers/google/cloud/_internal_client/secret_manager_client.py‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
import re
19+
from typing import Optional
20+
21+
import google
22+
from cached_property import cached_property
23+
from google.api_core.exceptions import NotFound
24+
from google.api_core.gapic_v1.client_info import ClientInfo
25+
from google.cloud.secretmanager_v1 import SecretManagerServiceClient
26+
27+
from airflow.utils.log.logging_mixin import LoggingMixin
28+
from airflow.version import version
29+
30+
SECRET_ID_PATTERN = r"^[a-zA-Z0-9-_]*$"
31+
32+
33+
class _SecretManagerClient(LoggingMixin):
34+
35+
"""
36+
Retrieves Secrets object from GCP Secrets Manager. This is a common class reused between SecretsManager
37+
and Secrets Hook that provides the shared authentication and verification mechanisms. This class should
38+
not be used directly, use SecretsManager or SecretsHook instead
39+
40+
41+
:param credentials: Credentials used to authenticate to GCP
42+
:type credentials: google.auth.credentials.Credentials
43+
"""
44+
def __init__(
45+
self,
46+
credentials: google.auth.credentials.Credentials,
47+
):
48+
super().__init__()
49+
self.credentials = credentials
50+
51+
@staticmethod
52+
def is_valid_secret_name(secret_name: str) -> bool:
53+
"""
54+
Returns true if the secret name is valid.
55+
:param secret_name: name of the secret
56+
:type secret_name: str
57+
:return:
58+
"""
59+
return bool(re.match(SECRET_ID_PATTERN, secret_name))
60+
61+
@cached_property
62+
def client(self) -> SecretManagerServiceClient:
63+
"""
64+
Create an authenticated KMS client
65+
"""
66+
_client = SecretManagerServiceClient(
67+
credentials=self.credentials,
68+
client_info=ClientInfo(client_library_version='airflow_v' + version)
69+
)
70+
return _client
71+
72+
def get_secret(self,
73+
secret_id: str,
74+
project_id: str,
75+
secret_version: str = 'latest') -> Optional[str]:
76+
"""
77+
Get secret value from the Secret Manager.
78+
79+
:param secret_id: Secret Key
80+
:type secret_id: str
81+
:param project_id: Project id to use
82+
:type project_id: str
83+
:param secret_version: version of the secret (default is 'latest')
84+
:type secret_version: str
85+
"""
86+
name = self.client.secret_version_path(project_id, secret_id, secret_version)
87+
try:
88+
response = self.client.access_secret_version(name)
89+
value = response.payload.data.decode('UTF-8')
90+
return value
91+
except NotFound:
92+
self.log.error(
93+
"GCP API Call Error (NotFound): Secret ID %s not found.", secret_id
94+
)
95+
return None

‎airflow/providers/google/cloud/hooks/secret_manager.py‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
"""Hook for Secrets Manager service"""
19+
from typing import Optional
20+
21+
from airflow.providers.google.cloud._internal_client.secret_manager_client import _SecretManagerClient # noqa
22+
from airflow.providers.google.common.hooks.base_google import GoogleBaseHook
23+
24+
25+
# noinspection PyAbstractClass
26+
class SecretsManagerHook(GoogleBaseHook):
27+
"""
28+
Hook for the Google Secret Manager API.
29+
30+
See https://cloud.google.com/secret-manager
31+
32+
All the methods in the hook where project_id is used must be called with
33+
keyword arguments rather than positional.
34+
35+
:param gcp_conn_id: The connection ID to use when fetching connection info.
36+
:type gcp_conn_id: str
37+
:param delegate_to: The account to impersonate, if any.
38+
For this to work, the service account making the request must have
39+
domain-wide delegation enabled.
40+
:type delegate_to: str
41+
"""
42+
def __init__(
43+
self,
44+
gcp_conn_id: str = "google_cloud_default",
45+
delegate_to: Optional[str] = None
46+
) -> None:
47+
super().__init__(gcp_conn_id, delegate_to)
48+
self.client = _SecretManagerClient(credentials=self._get_credentials())
49+
50+
def get_conn(self) -> _SecretManagerClient:
51+
"""
52+
Retrieves the connection to Secret Manager.
53+
54+
:return: Secret Manager client.
55+
:rtype: airflow.providers.google.cloud._internal_client.secret_manager_client._SecretManagerClient
56+
"""
57+
return self.client
58+
59+
@GoogleBaseHook.fallback_to_default_project_id
60+
def get_secret(self, secret_id: str,
61+
secret_version: str = 'latest',
62+
project_id: Optional[str] = None) -> Optional[str]:
63+
"""
64+
Get secret value from the Secret Manager.
65+
66+
:param secret_id: Secret Key
67+
:type secret_id: str
68+
:param secret_version: version of the secret (default is 'latest')
69+
:type secret_version: str
70+
:param project_id: Project id (if you want to override the project_id from credentials)
71+
:type project_id: str
72+
"""
73+
return self.get_conn().get_secret(secret_id=secret_id, secret_version=secret_version,
74+
project_id=project_id) # type: ignore

airflow/providers/google/cloud/secrets/secrets_manager.py renamed to airflow/providers/google/cloud/secrets/secret_manager.py

Lines changed: 32 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -18,26 +18,20 @@
1818
"""
1919
Objects relating to sourcing connections from GCP Secrets Manager
2020
"""
21-
import re
2221
from typing import Optional
2322

2423
from cached_property import cached_property
25-
from google.api_core.exceptions import NotFound
26-
from google.api_core.gapic_v1.client_info import ClientInfo
27-
from google.cloud.secretmanager_v1 import SecretManagerServiceClient
2824

29-
from airflow import version
3025
from airflow.exceptions import AirflowException
31-
from airflow.providers.google.cloud.utils.credentials_provider import (
32-
_get_scopes, get_credentials_and_project_id,
33-
)
26+
from airflow.providers.google.cloud._internal_client.secret_manager_client import _SecretManagerClient # noqa
27+
from airflow.providers.google.cloud.utils.credentials_provider import get_credentials_and_project_id
3428
from airflow.secrets import BaseSecretsBackend
3529
from airflow.utils.log.logging_mixin import LoggingMixin
3630

3731
SECRET_ID_PATTERN = r"^[a-zA-Z0-9-_]*$"
3832

3933

40-
class CloudSecretsManagerBackend(BaseSecretsBackend, LoggingMixin):
34+
class CloudSecretManagerBackend(BaseSecretsBackend, LoggingMixin):
4135
"""
4236
Retrieves Connection object from GCP Secrets Manager
4337
@@ -46,11 +40,11 @@ class CloudSecretsManagerBackend(BaseSecretsBackend, LoggingMixin):
4640
.. code-block:: ini
4741
4842
[secrets]
49-
backend = airflow.providers.google.cloud.secrets.secrets_manager.CloudSecretsManagerBackend
43+
backend = airflow.providers.google.cloud.secrets.secret_manager.CloudSecretManagerBackend
5044
backend_kwargs = {"connections_prefix": "airflow-connections", "sep": "-"}
5145
5246
For example, if the Secrets Manager secret id is ``airflow-connections-smtp_default``, this would be
53-
accessiblen if you provide ``{"connections_prefix": "airflow-connections", "sep": "-"}`` and request
47+
accessible if you provide ``{"connections_prefix": "airflow-connections", "sep": "-"}`` and request
5448
conn_id ``smtp_default``.
5549
5650
If the Secrets Manager secret id is ``airflow-variables-hello``, this would be
@@ -63,60 +57,63 @@ class CloudSecretsManagerBackend(BaseSecretsBackend, LoggingMixin):
6357
:type connections_prefix: str
6458
:param variables_prefix: Specifies the prefix of the secret to read to get Variables.
6559
:type variables_prefix: str
66-
:param gcp_key_path: Path to GCP Credential JSON file;
60+
:param gcp_key_path: Path to GCP Credential JSON file. Mutually exclusive with gcp_keyfile_dict.
6761
use default credentials in the current environment if not provided.
6862
:type gcp_key_path: str
63+
:param gcp_keyfile_dict: Dictionary of keyfile parameters. Mutually exclusive with gcp_key_path.
64+
:type gcp_keyfile_dict: dict
6965
:param gcp_scopes: Comma-separated string containing GCP scopes
7066
:type gcp_scopes: str
67+
:param project_id: Project id (if you want to override the project_id from credentials)
68+
:type project_id: str
7169
:param sep: separator used to concatenate connections_prefix and conn_id. Default: "-"
7270
:type sep: str
7371
"""
7472
def __init__(
7573
self,
7674
connections_prefix: str = "airflow-connections",
7775
variables_prefix: str = "airflow-variables",
76+
gcp_keyfile_dict: Optional[dict] = None,
7877
gcp_key_path: Optional[str] = None,
7978
gcp_scopes: Optional[str] = None,
79+
project_id: Optional[str] = None,
8080
sep: str = "-",
8181
**kwargs
8282
):
8383
super().__init__(**kwargs)
8484
self.connections_prefix = connections_prefix
8585
self.variables_prefix = variables_prefix
86-
self.gcp_key_path = gcp_key_path
87-
self.gcp_scopes = gcp_scopes
8886
self.sep = sep
89-
self.credentials: Optional[str] = None
90-
self.project_id: Optional[str] = None
9187
if not self._is_valid_prefix_and_sep():
9288
raise AirflowException(
9389
"`connections_prefix`, `variables_prefix` and `sep` should "
9490
f"follows that pattern {SECRET_ID_PATTERN}"
9591
)
96-
97-
def _is_valid_prefix_and_sep(self) -> bool:
98-
prefix = self.connections_prefix + self.sep
99-
return bool(re.match(SECRET_ID_PATTERN, prefix))
92+
self.credentials, self.project_id = get_credentials_and_project_id(
93+
keyfile_dict=gcp_keyfile_dict,
94+
key_path=gcp_key_path,
95+
scopes=gcp_scopes
96+
)
97+
# In case project id provided
98+
if project_id:
99+
self.project_id = project_id
100100

101101
@cached_property
102-
def client(self) -> SecretManagerServiceClient:
102+
def client(self) -> _SecretManagerClient:
103103
"""
104-
Create an authenticated KMS client
104+
Cached property returning secret client.
105+
106+
:return: Secrets client
105107
"""
106-
scopes = _get_scopes(self.gcp_scopes)
107-
self.credentials, self.project_id = get_credentials_and_project_id(
108-
key_path=self.gcp_key_path,
109-
scopes=scopes
110-
)
111-
_client = SecretManagerServiceClient(
112-
credentials=self.credentials,
113-
client_info=ClientInfo(client_library_version='airflow_v' + version.version)
114-
)
115-
return _client
108+
return _SecretManagerClient(credentials=self.credentials)
109+
110+
def _is_valid_prefix_and_sep(self) -> bool:
111+
prefix = self.connections_prefix + self.sep
112+
return _SecretManagerClient.is_valid_secret_name(prefix)
116113

117114
def get_conn_uri(self, conn_id: str) -> Optional[str]:
118115
"""
119-
Get secret value from Secrets Manager.
116+
Get secret value from the SecretManager.
120117
121118
:param conn_id: connection id
122119
:type conn_id: str
@@ -134,23 +131,12 @@ def get_variable(self, key: str) -> Optional[str]:
134131

135132
def _get_secret(self, path_prefix: str, secret_id: str) -> Optional[str]:
136133
"""
137-
Get secret value from Parameter Store.
134+
Get secret value from the SecretManager based on prefix.
138135
139136
:param path_prefix: Prefix for the Path to get Secret
140137
:type path_prefix: str
141138
:param secret_id: Secret Key
142139
:type secret_id: str
143140
"""
144141
secret_id = self.build_path(path_prefix, secret_id, self.sep)
145-
# always return the latest version of the secret
146-
secret_version = "latest"
147-
name = self.client.secret_version_path(self.project_id, secret_id, secret_version)
148-
try:
149-
response = self.client.access_secret_version(name)
150-
value = response.payload.data.decode('UTF-8')
151-
return value
152-
except NotFound:
153-
self.log.error(
154-
"GCP API Call Error (NotFound): Secret ID %s not found.", secret_id
155-
)
156-
return None
142+
return self.client.get_secret(secret_id=secret_id, project_id=self.project_id)

‎docs/conf.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -203,8 +203,10 @@
203203
"_api/airflow/providers/cncf/index.rst",
204204
# Utils for internal use
205205
'_api/airflow/providers/google/cloud/utils',
206-
# Internal client for hashicorp
206+
# Internal client for Hashicorp Vault
207207
'_api/airflow/providers/hashicorp/_internal_client',
208+
# Internal client for GCP Secret Manager
209+
'_api/airflow/providers/google/cloud/_internal_client',
208210
# Templates or partials
209211
'autoapi_templates',
210212
'howto/operator/gcp/_partials',

0 commit comments

Comments
 (0)