Skip to content

Commit

Permalink
Fix error with quick-failing tasks in KubernetesPodOperator (#13621)
Browse files Browse the repository at this point in the history
* Fix error with quick-failing tasks in KubernetesPodOperator

Addresses an issue with the KubernetesPodOperator where tasks that die
quickly are not patched with "already_checked" because they never make
it to the monitoring logic.

* static fix
  • Loading branch information
dimberman committed Jan 21, 2021
1 parent 10b8ecc commit 94d3ed6
Show file tree
Hide file tree
Showing 2 changed files with 26 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,7 @@ def execute(self, context) -> Optional[str]:
raise AirflowException(f'Pod {self.pod.metadata.name} returned a failure: {status}')
return result
except AirflowException as ex:
self.patch_already_checked(self.pod)
raise AirflowException(f'Pod Launching failed: {ex}')

def handle_pod_overlap(
Expand Down
25 changes: 25 additions & 0 deletions kubernetes_tests/test_kubernetes_pod_operator.py
Original file line number Diff line number Diff line change
Expand Up @@ -1000,5 +1000,30 @@ def test_reattach_failing_pod_once(self):
k.execute(context)
create_mock.assert_called_once()

def test_reatttach_quick_failure(self):
client = kube_client.get_kube_client(in_cluster=False)
namespace = "default"

name = "test"
k = KubernetesPodOperator(
namespace='default',
image="ubuntu:16.04",
cmds=["bash", "-cx"],
arguments=["exit 1"],
labels={"foo": "bar"},
name="test",
task_id=name,
in_cluster=False,
do_xcom_push=False,
is_delete_operator_pod=False,
termination_grace_period=0,
)

context = create_context(k)
with self.assertRaises(AirflowException):
k.execute(context)
pod = client.read_namespaced_pod(name=k.pod.metadata.name, namespace=namespace)
self.assertEqual(pod.metadata.labels["already_checked"], "True")


# pylint: enable=unused-argument

0 comments on commit 94d3ed6

Please sign in to comment.