Skip to content

Commit fa96b09

Browse files
ashbashb
authored
Update Sphinx and Sphinx-AutoAPI (#20079)
We were stuck on an old version of Sphinx AutoAPI for a long while as more recent versions wouldn't build Airflow's docs, but that seems to have finally been resolved. We can remove the run_patched_sphinx.py as that was included in sphinx-autoapi 1.1 * Fix doc rendering glitch in Google provider utils * Remove duplicated link from cncf-kubernetes provider index
1 parent 820e836 commit fa96b09

13 files changed

Lines changed: 75 additions & 155 deletions

File tree

‎airflow/models/dag.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2961,7 +2961,7 @@ def calculate_dagrun_date_fields(
29612961
def dag(*dag_args, **dag_kwargs):
29622962
"""
29632963
Python dag decorator. Wraps a function into an Airflow DAG.
2964-
Accepts kwargs for operator kwarg. Can be used to parametrize DAGs.
2964+
Accepts kwargs for operator kwarg. Can be used to parameterize DAGs.
29652965
29662966
:param dag_args: Arguments for DAG object
29672967
:type dag_args: Any
@@ -3021,6 +3021,7 @@ def factory(*args, **kwargs):
30213021
from airflow.models.serialized_dag import SerializedDagModel
30223022

30233023
DagModel.serialized_dag = relationship(SerializedDagModel)
3024+
""":sphinx-autoapi-skip:"""
30243025

30253026

30263027
class DagContext:

‎airflow/providers/google/cloud/utils/credentials_provider.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,8 @@ def build_gcp_conn(
7979
@contextmanager
8080
def provide_gcp_credentials(key_file_path: Optional[str] = None, key_file_dict: Optional[Dict] = None):
8181
"""
82-
Context manager that provides a Google Cloud credentials for application supporting `Application
83-
Default Credentials (ADC) strategy <https://cloud.google.com/docs/authentication/production>`__.
82+
Context manager that provides a Google Cloud credentials for application supporting
83+
`Application Default Credentials (ADC) strategy`__.
8484
8585
It can be used to provide credentials for external programs (e.g. gcloud) that expect authorization
8686
file in ``GOOGLE_APPLICATION_CREDENTIALS`` environment variable.
@@ -89,6 +89,8 @@ def provide_gcp_credentials(key_file_path: Optional[str] = None, key_file_dict:
8989
:type key_file_path: str
9090
:param key_file_dict: Dictionary with credentials.
9191
:type key_file_dict: Dict
92+
93+
__ https://cloud.google.com/docs/authentication/production
9294
"""
9395
if not key_file_path and not key_file_dict:
9496
raise ValueError("Please provide `key_file_path` or `key_file_dict`.")
@@ -146,7 +148,7 @@ def provide_gcp_conn_and_credentials(
146148
Context manager that provides both:
147149
148150
- Google Cloud credentials for application supporting `Application Default Credentials (ADC)
149-
strategy <https://cloud.google.com/docs/authentication/production>`__.
151+
strategy`__.
150152
- temporary value of :envvar:`AIRFLOW_CONN_GOOGLE_CLOUD_DEFAULT` connection
151153
152154
:param key_file_path: Path to file with Google Cloud Service Account .json file.
@@ -155,6 +157,8 @@ def provide_gcp_conn_and_credentials(
155157
:type scopes: Sequence
156158
:param project_id: The id of Google Cloud project for the connection.
157159
:type project_id: str
160+
161+
__ https://cloud.google.com/docs/authentication/production
158162
"""
159163
with ExitStack() as stack:
160164
if key_file_path:

‎airflow/providers/google/cloud/utils/mlengine_prediction_summary.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,10 @@ def metric_fn(inst):
101101
"--temp_location=gs://...",
102102
]
103103
)
104+
105+
.. spelling::
106+
107+
pcoll
104108
"""
105109

106110
import argparse

‎airflow/providers/google/common/utils/id_token_credentials.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@
2323
2424
ID_TOKEN="$(python -m airflow.providers.google.common.utils.id_token_credentials)"
2525
curl "https://www.googleapis.com/oauth2/v3/tokeninfo?id_token=${ID_TOKEN}" -v
26+
27+
.. spelling::
28+
29+
RefreshError
2630
"""
2731

2832
import json

‎airflow/sensors/base.py‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818

1919
import datetime
2020
import hashlib
21-
import os
2221
import time
2322
from datetime import timedelta
2423
from typing import Any, Callable, Dict, Iterable
@@ -39,7 +38,7 @@
3938
# We need to keep the import here because GCSToLocalFilesystemOperator released in
4039
# Google Provider before 3.0.0 imported apply_defaults from here.
4140
# See https://github.com/apache/airflow/issues/16035
42-
from airflow.utils.decorators import apply_defaults
41+
from airflow.utils.decorators import apply_defaults # noqa: F401
4342

4443

4544
class BaseSensorOperator(BaseOperator, SkipMixin):
@@ -122,7 +121,8 @@ def _validate_input_values(self) -> None:
122121
raise AirflowException("The timeout must be a non-negative number")
123122
if self.mode not in self.valid_modes:
124123
raise AirflowException(
125-
f"The mode must be one of {self.valid_modes},'{self.dag.dag_id if self.has_dag() else ''}.{self.task_id}'; received '{self.mode}'."
124+
f"The mode must be one of {self.valid_modes},'{self.dag.dag_id if self.has_dag() else ''} "
125+
f".{self.task_id}'; received '{self.mode}'."
126126
)
127127

128128
def poke(self, context: Dict) -> bool:
@@ -318,9 +318,3 @@ def mode_setter(_, value):
318318
return cls_type
319319

320320
return decorate(cls)
321-
322-
323-
if 'BUILDING_AIRFLOW_DOCS' in os.environ:
324-
# flake8: noqa: F811
325-
# Monkey patch hook to get good function headers while building docs
326-
apply_defaults = lambda x: x

‎docs/apache-airflow-providers-cncf-kubernetes/index.rst‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ Content
4242
Example DAGs <https://github.com/apache/airflow/tree/main/airflow/providers/cncf/kubernetes/example_dags>
4343
PyPI Repository <https://pypi.org/project/apache-airflow-providers-cncf-kubernetes/>
4444
Installing from sources <installing-providers-from-sources>
45-
Installing from sources <installing-providers-from-sources>
4645

4746
.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME!
4847

‎docs/apache-airflow/executor/kubernetes.rst‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,21 +100,21 @@ With these requirements in mind, here are some examples of basic ``pod_template_
100100

101101
Storing DAGs in the image:
102102

103-
.. exampleinclude:: /../../airflow/kubernetes/pod_template_file_examples/dags_in_image_template.yaml
103+
.. literalinclude:: /../../airflow/kubernetes/pod_template_file_examples/dags_in_image_template.yaml
104104
:language: yaml
105105
:start-after: [START template_with_dags_in_image]
106106
:end-before: [END template_with_dags_in_image]
107107

108108
Storing DAGs in a ``persistentVolume``:
109109

110-
.. exampleinclude:: /../../airflow/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml
110+
.. literalinclude:: /../../airflow/kubernetes/pod_template_file_examples/dags_in_volume_template.yaml
111111
:language: yaml
112112
:start-after: [START template_with_dags_in_volume]
113113
:end-before: [END template_with_dags_in_volume]
114114

115115
Pulling DAGs from ``git``:
116116

117-
.. exampleinclude:: /../../airflow/kubernetes/pod_template_file_examples/git_sync_template.yaml
117+
.. literalinclude:: /../../airflow/kubernetes/pod_template_file_examples/git_sync_template.yaml
118118
:language: yaml
119119
:start-after: [START git_sync_template]
120120
:end-before: [END git_sync_template]

‎docs/conf.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,9 @@
121121
.. |experimental| replace:: This is an :ref:`experimental feature <experimental>`.
122122
"""
123123

124+
smartquotes_excludes = {'builders': ['man', 'text', 'spelling']}
125+
126+
124127
# -- General configuration -----------------------------------------------------
125128
# See: https://www.sphinx-doc.org/en/master/usage/configuration.html
126129

@@ -674,6 +677,8 @@ def _get_params(root_schema: dict, prefix: str = "", default_section: str = "")
674677
]
675678
if PACKAGE_NAME == 'apache-airflow':
676679
autoapi_ignore.append('*/airflow/providers/*')
680+
else:
681+
autoapi_ignore.append('*/airflow/providers/cncf/kubernetes/backcompat/*')
677682
# Keep the AutoAPI generated files on the filesystem after the run.
678683
# Useful for debugging.
679684
autoapi_keep_files = True
@@ -696,6 +701,10 @@ def _get_params(root_schema: dict, prefix: str = "", default_section: str = "")
696701
'special-members',
697702
]
698703

704+
suppress_warnings = [
705+
"autoapi.python_import_resolution",
706+
]
707+
699708
# -- Options for ext.exampleinclude --------------------------------------------
700709
exampleinclude_sourceroot = os.path.abspath('..')
701710

@@ -709,6 +718,7 @@ def _get_params(root_schema: dict, prefix: str = "", default_section: str = "")
709718
if PACKAGE_NAME == 'helm-chart':
710719
spelling_exclude_patterns = ['changelog.rst']
711720
spelling_ignore_contributor_names = False
721+
spelling_ignore_importable_modules = True
712722

713723
# -- Options for sphinxcontrib.redoc -------------------------------------------
714724
# See: https://sphinxcontrib-redoc.readthedocs.io/en/stable/
@@ -730,3 +740,14 @@ def _get_params(root_schema: dict, prefix: str = "", default_section: str = "")
730740

731741
# Options for script updater
732742
redoc_script_url = "https://cdn.jsdelivr.net/npm/redoc@2.0.0-rc.48/bundles/redoc.standalone.js"
743+
744+
745+
def skip_util_classes(app, what, name, obj, skip, options):
746+
if (what == "data" and "STATICA_HACK" in name) or ":sphinx-autoapi-skip:" in obj.docstring:
747+
skip = True
748+
return skip
749+
750+
751+
def setup(sphinx):
752+
if 'autoapi.extension' in extensions:
753+
sphinx.connect("autoapi-skip-member", skip_util_classes)

‎docs/exts/docs_build/docs_builder.py‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@
3030
CONSOLE_WIDTH,
3131
DOCS_DIR,
3232
PROCESS_TIMEOUT,
33-
ROOT_PROJECT_DIR,
3433
pretty_format_path,
3534
)
3635
from docs.exts.docs_build.errors import DocBuildError, parse_sphinx_warnings
@@ -138,7 +137,7 @@ def check_spelling(self, verbose: bool) -> List[SpellingError]:
138137
os.makedirs(self.log_spelling_output_dir, exist_ok=True)
139138

140139
build_cmd = [
141-
os.path.join(ROOT_PROJECT_DIR, "docs", "exts", "docs_build", "run_patched_sphinx.py"),
140+
"sphinx-build",
142141
"-W", # turn warnings into errors
143142
"--color", # do emit colored output
144143
"-T", # show full traceback on exception
@@ -213,7 +212,7 @@ def build_sphinx_docs(self, verbose: bool) -> List[DocBuildError]:
213212
os.makedirs(self._build_dir, exist_ok=True)
214213

215214
build_cmd = [
216-
os.path.join(ROOT_PROJECT_DIR, "docs", "exts", "docs_build", "run_patched_sphinx.py"),
215+
"sphinx-build",
217216
"-T", # show full traceback on exception
218217
"--color", # do emit colored output
219218
"-b", # builder to use

‎docs/exts/docs_build/run_patched_sphinx.py‎

Lines changed: 0 additions & 125 deletions
This file was deleted.

0 commit comments

Comments
 (0)