More strict rules in mypy (#9705) (#9906)

retornam · web-flow · commit c2db0dfeb13e · 2020-07-22T13:56:02.000+02:00
Signed-off-by: Raymond Etornam &lt;retornam@users.noreply.github.com&gt;
diff --git a/airflow/cli/cli_parser.py b/airflow/cli/cli_parser.py
@@ -51,7 +51,7 @@ def command(*args, **kwargs):
         func = import_string(import_path)
         return func(*args, **kwargs)
 
-    command.__name__ = name  # type: ignore
+    command.__name__ = name
 
     return command
 
diff --git a/airflow/configuration.py b/airflow/configuration.py
@@ -467,7 +467,7 @@ def getsection(self, section: str) -> Optional[Dict[str, Union[str, int, float,
         if (section not in self._sections and section not in self.airflow_defaults._sections):  # type: ignore
             return None
 
-        _section = copy.deepcopy(self.airflow_defaults._sections[section])  # type: ignore
+        _section = copy.deepcopy(self.airflow_defaults._sections[section])
 
         if section in self._sections:  # type: ignore
             _section.update(copy.deepcopy(self._sections[section]))  # type: ignore
@@ -481,7 +481,7 @@ def getsection(self, section: str) -> Optional[Dict[str, Union[str, int, float,
                 key = key.lower()
                 _section[key] = self._get_env_var_option(section, key)
 
-        for key, val in _section.items():  # type: ignore
+        for key, val in _section.items():
             try:
                 val = int(val)
             except ValueError:
@@ -499,13 +499,13 @@ def write(self, fp, space_around_delimiters=True):
         # This is based on the configparser.RawConfigParser.write method code to add support for
         # reading options from environment variables.
         if space_around_delimiters:
-            d = " {} ".format(self._delimiters[0])  # type: ignore
+            d = " {} ".format(self._delimiters[0])
         else:
-            d = self._delimiters[0]  # type: ignore
+            d = self._delimiters[0]
         if self._defaults:
-            self._write_section(fp, self.default_section, self._defaults.items(), d)  # type: ignore
+            self._write_section(fp, self.default_section, self._defaults.items(), d)
         for section in self._sections:
-            self._write_section(fp, section, self.getsection(section).items(), d)  # type: ignore
+            self._write_section(fp, section, self.getsection(section).items(), d)
 
     def as_dict(
             self, display_source=False, display_sensitive=False, raw=False,
diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py
@@ -26,9 +26,7 @@
 import warnings
 from abc import ABCMeta, abstractmethod
 from datetime import datetime, timedelta
-from typing import (
-    Any, Callable, ClassVar, Dict, FrozenSet, Iterable, List, Optional, Set, Tuple, Type, Union, cast,
-)
+from typing import Any, Callable, ClassVar, Dict, FrozenSet, Iterable, List, Optional, Set, Tuple, Type, Union
 
 import attr
 import jinja2
@@ -1168,7 +1166,7 @@ def _set_relatives(self,
                 task_list = [task_or_task_list]  # type: ignore
 
             task_list = [
-                t.operator if isinstance(t, XComArg) else t  # type: ignore
+                t.operator if isinstance(t, XComArg) else t
                 for t in task_list
             ]
 
@@ -1381,8 +1379,8 @@ def chain(*tasks: Union[BaseOperator, List[BaseOperator]]):
             raise TypeError(
                 'Chain not supported between instances of {up_type} and {down_type}'.format(
                     up_type=type(up_task), down_type=type(down_task)))
-        up_task_list = cast(List[BaseOperator], up_task)
-        down_task_list = cast(List[BaseOperator], down_task)
+        up_task_list = up_task
+        down_task_list = down_task
         if len(up_task_list) != len(down_task_list):
             raise AirflowException(
                 f'Chain not supported different length Iterable '
diff --git a/airflow/models/dagrun.py b/airflow/models/dagrun.py
@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 from datetime import datetime
-from typing import Any, List, Optional, Tuple, Union, cast
+from typing import Any, List, Optional, Tuple, Union
 
 from sqlalchemy import (
     Boolean, Column, DateTime, Index, Integer, PickleType, String, UniqueConstraint, and_, func, or_,
@@ -266,8 +266,6 @@ def get_dag(self):
     def get_previous_dagrun(self, state: Optional[str] = None, session: Session = None) -> Optional['DagRun']:
         """The previous DagRun, if there is one"""
 
-        session = cast(Session, session)  # mypy
-
         filters = [
             DagRun.dag_id == self.dag_id,
             DagRun.execution_date < self.execution_date,
diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py
@@ -1771,9 +1771,9 @@ def filter_for_tis(
                                for tik in tis])
             return or_(*filter_for_tis)
         if all(isinstance(t, TaskInstance) for t in tis):
-            filter_for_tis = ([and_(TI.dag_id == ti.dag_id,  # type: ignore
-                                    TI.task_id == ti.task_id,  # type: ignore
-                                    TI.execution_date == ti.execution_date)  # type: ignore
+            filter_for_tis = ([and_(TI.dag_id == ti.dag_id,
+                                    TI.task_id == ti.task_id,
+                                    TI.execution_date == ti.execution_date)
                                for ti in tis])
             return or_(*filter_for_tis)
 
diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py
@@ -29,8 +29,8 @@
 
 import pkg_resources
 
-from airflow import settings  # type: ignore
-from airflow.utils.file import find_path_from_directory  # type: ignore
+from airflow import settings
+from airflow.utils.file import find_path_from_directory
 
 log = logging.getLogger(__name__)
 
diff --git a/airflow/providers/apache/hive/transfers/mssql_to_hive.py b/airflow/providers/apache/hive/transfers/mssql_to_hive.py
@@ -119,7 +119,7 @@ def execute(self, context: Dict[str, str]):
                 cursor.execute(self.sql)
                 with NamedTemporaryFile("w") as tmp_file:
                     csv_writer = csv.writer(tmp_file, delimiter=self.delimiter, encoding='utf-8')
-                    field_dict = OrderedDict()  # type:ignore
+                    field_dict = OrderedDict()
                     col_count = 0
                     for field in cursor.description:
                         col_count += 1
diff --git a/airflow/providers/apache/hive/transfers/mysql_to_hive.py b/airflow/providers/apache/hive/transfers/mysql_to_hive.py
@@ -150,7 +150,7 @@ def execute(self, context: Dict[str, str]):
                                     quotechar=self.quotechar,
                                     escapechar=self.escapechar,
                                     encoding="utf-8")
-            field_dict = OrderedDict()  # type:ignore
+            field_dict = OrderedDict()
             for field in cursor.description:
                 field_dict[field[0]] = self.type_map(field[1])
             csv_writer.writerows(cursor)
diff --git a/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py b/airflow/providers/cncf/kubernetes/operators/kubernetes_pod.py
@@ -370,9 +370,9 @@ def create_new_pod_for_operator(self, labels, launcher) -> Tuple[State, k8s.V1Po
         # noinspection PyTypeChecker
         pod = append_to_pod(
             pod,
-            self.pod_runtime_info_envs +  # type: ignore
+            self.pod_runtime_info_envs +
             self.ports +  # type: ignore
-            self.resources +  # type: ignore
+            self.resources +
             self.secrets +  # type: ignore
             self.volumes +  # type: ignore
             self.volume_mounts  # type: ignore
diff --git a/airflow/providers/docker/operators/docker.py b/airflow/providers/docker/operators/docker.py
@@ -327,7 +327,7 @@ def __get_tls_config(self):
                 ca_cert=self.tls_ca_cert,
                 client_cert=(self.tls_client_cert, self.tls_client_key),
                 verify=True,
-                ssl_version=self.tls_ssl_version,  # type: ignore
+                ssl_version=self.tls_ssl_version,
                 assert_hostname=self.tls_hostname
             )
             self.docker_url = self.docker_url.replace('tcp://', 'https://')
diff --git a/airflow/providers/google/cloud/hooks/cloud_sql.py b/airflow/providers/google/cloud/hooks/cloud_sql.py
@@ -142,7 +142,7 @@ def create_instance(self, body: Dict, project_id: str) -> None:
             body=body
         ).execute(num_retries=self.num_retries)
         operation_name = response["name"]
-        self._wait_for_operation_to_complete(project_id=project_id,  # type:ignore
+        self._wait_for_operation_to_complete(project_id=project_id,
                                              operation_name=operation_name)
 
     @GoogleBaseHook.fallback_to_default_project_id
@@ -170,7 +170,7 @@ def patch_instance(self, body: Dict, instance: str, project_id: str) -> None:
             body=body
         ).execute(num_retries=self.num_retries)
         operation_name = response["name"]
-        self._wait_for_operation_to_complete(project_id=project_id,  # type:ignore
+        self._wait_for_operation_to_complete(project_id=project_id,
                                              operation_name=operation_name)
 
     @GoogleBaseHook.fallback_to_default_project_id
@@ -191,7 +191,7 @@ def delete_instance(self, instance: str, project_id: str) -> None:
             instance=instance,
         ).execute(num_retries=self.num_retries)
         operation_name = response["name"]
-        self._wait_for_operation_to_complete(project_id=project_id,  # type:ignore
+        self._wait_for_operation_to_complete(project_id=project_id,
                                              operation_name=operation_name)
 
     @GoogleBaseHook.fallback_to_default_project_id
@@ -238,7 +238,7 @@ def create_database(self, instance: str, body: Dict, project_id: str) -> None:
             body=body
         ).execute(num_retries=self.num_retries)
         operation_name = response["name"]
-        self._wait_for_operation_to_complete(project_id=project_id,  # type:ignore
+        self._wait_for_operation_to_complete(project_id=project_id,
                                              operation_name=operation_name)
 
     @GoogleBaseHook.fallback_to_default_project_id
@@ -275,7 +275,7 @@ def patch_database(
             body=body
         ).execute(num_retries=self.num_retries)
         operation_name = response["name"]
-        self._wait_for_operation_to_complete(project_id=project_id,  # type:ignore
+        self._wait_for_operation_to_complete(project_id=project_id,
                                              operation_name=operation_name)
 
     @GoogleBaseHook.fallback_to_default_project_id
@@ -299,7 +299,7 @@ def delete_database(self, instance: str, database: str, project_id: str) -> None
             database=database
         ).execute(num_retries=self.num_retries)
         operation_name = response["name"]
-        self._wait_for_operation_to_complete(project_id=project_id,  # type:ignore
+        self._wait_for_operation_to_complete(project_id=project_id,
                                              operation_name=operation_name)
 
     @GoogleBaseHook.fallback_to_default_project_id
@@ -326,7 +326,7 @@ def export_instance(self, instance: str, body: Dict, project_id: str) -> None:
             body=body
         ).execute(num_retries=self.num_retries)
         operation_name = response["name"]
-        self._wait_for_operation_to_complete(project_id=project_id,  # type:ignore
+        self._wait_for_operation_to_complete(project_id=project_id,
                                              operation_name=operation_name)
 
     @GoogleBaseHook.fallback_to_default_project_id
@@ -353,7 +353,7 @@ def import_instance(self, instance: str, body: Dict, project_id: str) -> None:
                 body=body
             ).execute(num_retries=self.num_retries)
             operation_name = response["name"]
-            self._wait_for_operation_to_complete(project_id=project_id,  # type: ignore
+            self._wait_for_operation_to_complete(project_id=project_id,
                                                  operation_name=operation_name)
         except HttpError as ex:
             raise AirflowException(
@@ -984,7 +984,7 @@ def cleanup_database_hook(self) -> None:
                 raise ValueError("The db_hook should be set")
             if not isinstance(self.db_hook, PostgresHook):
                 raise ValueError(f"The db_hook should be PostrgresHook and is {type(self.db_hook)}")
-            conn = getattr(self.db_hook, 'conn')  # type: ignore
+            conn = getattr(self.db_hook, 'conn')
             if conn and conn.notices:
                 for output in self.db_hook.conn.notices:
                     self.log.info(output)
diff --git a/airflow/providers/google/cloud/hooks/translate.py b/airflow/providers/google/cloud/hooks/translate.py
@@ -107,7 +107,7 @@ def translate(
         """
         client = self.get_conn()
 
-        return client.translate(  # type: ignore
+        return client.translate(
             values=values,
             target_language=target_language,
             format_=format_,
diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py
@@ -153,7 +153,7 @@ def __init__(
         super().__init__(sql=sql, *args, **kwargs)
         if bigquery_conn_id:
             warnings.warn(_DEPRECATION_MSG, DeprecationWarning, stacklevel=3)
-            gcp_conn_id = bigquery_conn_id  # type: ignore
+            gcp_conn_id = bigquery_conn_id
 
         self.gcp_conn_id = gcp_conn_id
         self.sql = sql
diff --git a/airflow/providers/google/marketing_platform/operators/campaign_manager.py b/airflow/providers/google/marketing_platform/operators/campaign_manager.py
@@ -314,7 +314,7 @@ def execute(self, context: Dict):
         self.log.info("Inserting Campaign Manager report.")
         response = hook.insert_report(
             profile_id=self.profile_id, report=self.report
-        )  # type: ignore
+        )
         report_id = response.get("id")
         self.xcom_push(context, key="report_id", value=report_id)
         self.log.info("Report successfully inserted. Report id: %s", report_id)
diff --git a/airflow/utils/file.py b/airflow/utils/file.py
@@ -22,7 +22,7 @@
 import zipfile
 from typing import Dict, Generator, List, Optional, Pattern
 
-from airflow.configuration import conf  # type: ignore
+from airflow.configuration import conf
 
 log = logging.getLogger(__name__)
 
@@ -159,7 +159,7 @@ def list_py_file_paths(directory: str,
     elif os.path.isdir(directory):
         find_dag_file_paths(directory, file_paths, safe_mode)
     if include_examples:
-        from airflow import example_dags  # type: ignore
+        from airflow import example_dags
         example_dag_folder = example_dags.__path__[0]  # type: ignore
         file_paths.extend(list_py_file_paths(example_dag_folder, safe_mode, False))
     return file_paths
diff --git a/docs/exts/exampleinclude.py b/docs/exts/exampleinclude.py
@@ -140,7 +140,7 @@ def register_source(app, env, modname):
     :param modname: name of the module to load
     :return: True if the code is registered successfully, False otherwise
     """
-    entry = env._viewcode_modules.get(modname, None)  # type: ignore
+    entry = env._viewcode_modules.get(modname, None)
     if entry is False:
         print("[%s] Entry is false for " % modname)
         return False
@@ -153,7 +153,7 @@ def register_source(app, env, modname):
         except Exception as ex:  # pylint: disable=broad-except
             logger.info("Module \"%s\" could not be loaded. Full source will not be available. \"%s\"",
                         modname, ex)
-            env._viewcode_modules[modname] = False  # type: ignore
+            env._viewcode_modules[modname] = False
             return False
 
         if not isinstance(analyzer.code, str):
@@ -169,7 +169,7 @@ def register_source(app, env, modname):
 
     if entry is None or entry[0] != code:
         entry = code, tags, {}, ""
-        env._viewcode_modules[modname] = entry  # type: ignore
+        env._viewcode_modules[modname] = entry
 
     return True
 # pylint: enable=protected-access
@@ -222,7 +222,7 @@ def doctree_read(app, doctree):
     """
     env = app.builder.env
     if not hasattr(env, "_viewcode_modules"):
-        env._viewcode_modules = {}  # type: ignore
+        env._viewcode_modules = {}
 
     if app.builder.name == "singlehtml":
         return
diff --git a/scripts/perf/sql_queries.py b/scripts/perf/sql_queries.py
@@ -200,7 +200,7 @@ def main() -> None:
         times.append(exec_time)
         for qry in queries:
             info = qry.to_dict()
-            info["test_no"] = i  # type: ignore
+            info["test_no"] = i
             rows.append(info)
 
     rows_to_csv(rows, name="/files/sql_after_remote.csv")
diff --git a/setup.cfg b/setup.cfg
@@ -58,6 +58,8 @@ packages = airflow
 [mypy]
 ignore_missing_imports = True
 no_implicit_optional = True
+warn_redundant_casts = True
+warn_unused_ignores = True
 plugins =
   airflow.mypy.plugin.decorators
 pretty = True
diff --git a/tests/api_connexion/endpoints/test_dag_endpoint.py b/tests/api_connexion/endpoints/test_dag_endpoint.py
@@ -127,9 +127,9 @@ def test_should_response_200(self):
 
     def test_should_response_200_serialized(self):
         # Create empty app with empty dagbag to check if DAG is read from db
-        app_serialized = app.create_app(testing=True)  # type:ignore
+        app_serialized = app.create_app(testing=True)
         dag_bag = DagBag(os.devnull, include_examples=False, read_dags_from_db=True)
-        app_serialized.dag_bag = dag_bag  # type:ignore
+        app_serialized.dag_bag = dag_bag
         client = app_serialized.test_client()
 
         SerializedDagModel.write_dag(self.dag)
diff --git a/tests/api_connexion/endpoints/test_task_endpoint.py b/tests/api_connexion/endpoints/test_task_endpoint.py
@@ -94,9 +94,9 @@ def test_should_response_200(self):
     @conf_vars({("core", "store_serialized_dags"): "True"})
     def test_should_response_200_serialized(self):
         # Create empty app with empty dagbag to check if DAG is read from db
-        app_serialized = app.create_app(testing=True)  # type:ignore
+        app_serialized = app.create_app(testing=True)
         dag_bag = DagBag(os.devnull, include_examples=False, read_dags_from_db=True)
-        app_serialized.dag_bag = dag_bag  # type:ignore
+        app_serialized.dag_bag = dag_bag
         client = app_serialized.test_client()
 
         SerializedDagModel.write_dag(self.dag)
diff --git a/tests/cli/commands/test_task_command.py b/tests/cli/commands/test_task_command.py
@@ -163,7 +163,7 @@ def test_cli_run_mutually_exclusive(self):
             AirflowException,
             "Option --raw and --local are mutually exclusive."
         ):
-            task_command.task_run(self.parser.parse_args([  # type: ignore
+            task_command.task_run(self.parser.parse_args([
                 'tasks', 'run', 'example_bash_operator', 'runme_0', DEFAULT_DATE.isoformat(), '--raw',
                 '--local'
             ]))
diff --git a/tests/plugins/test_plugin_ignore.py b/tests/plugins/test_plugin_ignore.py
@@ -23,8 +23,8 @@
 import unittest
 from unittest.mock import patch
 
-from airflow import settings  # type: ignore
-from airflow.utils.file import find_path_from_directory  # type: ignore
+from airflow import settings
+from airflow.utils.file import find_path_from_directory
 
 
 class TestIgnorePluginFile(unittest.TestCase):
diff --git a/tests/providers/google/cloud/hooks/test_cloud_memorystore.py b/tests/providers/google/cloud/hooks/test_cloud_memorystore.py
diff --git a/tests/providers/google/cloud/hooks/test_datacatalog.py b/tests/providers/google/cloud/hooks/test_datacatalog.py
diff --git a/tests/providers/google/cloud/hooks/test_dlp.py b/tests/providers/google/cloud/hooks/test_dlp.py
diff --git a/tests/providers/google/cloud/hooks/test_tasks.py b/tests/providers/google/cloud/hooks/test_tasks.py
diff --git a/tests/providers/google/cloud/log/test_stackdriver_task_handler.py b/tests/providers/google/cloud/log/test_stackdriver_task_handler.py
diff --git a/tests/providers/google/common/hooks/test_base_google.py b/tests/providers/google/common/hooks/test_base_google.py

Original file line number	Diff line number	Diff line change
`@@ -327,7 +327,7 @@ def __get_tls_config(self):`
`327`	`327`	`ca_cert=self.tls_ca_cert,`
`328`	`328`	`client_cert=(self.tls_client_cert, self.tls_client_key),`
`329`	`329`	`verify=True,`
`330`		`- ssl_version=self.tls_ssl_version, # type: ignore`
	`330`	`+ ssl_version=self.tls_ssl_version,`
`331`	`331`	`assert_hostname=self.tls_hostname`
`332`	`332`	`)`
`333`	`333`	`self.docker_url = self.docker_url.replace('tcp://', 'https://')`