recovery_alert.py 4.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107
  1. #!/usr/bin/env python
  2. """
  3. Licensed to the Apache Software Foundation (ASF) under one
  4. or more contributor license agreements. See the NOTICE file
  5. distributed with this work for additional information
  6. regarding copyright ownership. The ASF licenses this file
  7. to you under the Apache License, Version 2.0 (the
  8. "License"); you may not use this file except in compliance
  9. with the License. You may obtain a copy of the License at
  10. http://www.apache.org/licenses/LICENSE-2.0
  11. Unless required by applicable law or agreed to in writing, software
  12. distributed under the License is distributed on an "AS IS" BASIS,
  13. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14. See the License for the specific language governing permissions and
  15. limitations under the License.
  16. """
  17. import logging
  18. import datetime
  19. from alerts.base_alert import BaseAlert
  20. logger = logging.getLogger()
  21. # default recoveries counts
  22. DEFAULT_WARNING_RECOVERIES_COUNT = 2
  23. DEFAULT_CRITICAL_RECOVERIES_COUNT = 4
  24. UNKNOWN_COMPONENT = 'UNKNOWN_COMPONENT'
  25. class RecoveryAlert(BaseAlert):
  26. def __init__(self, alert_meta, alert_source_meta, recovery_manager):
  27. super(RecoveryAlert, self).__init__(alert_meta, alert_source_meta)
  28. self.recovery_manager = recovery_manager
  29. self.warning_count = DEFAULT_WARNING_RECOVERIES_COUNT
  30. self.critical_count = DEFAULT_CRITICAL_RECOVERIES_COUNT
  31. if 'reporting' in alert_source_meta:
  32. reporting = alert_source_meta['reporting']
  33. reporting_state_warning = self.RESULT_WARNING.lower()
  34. reporting_state_critical = self.RESULT_CRITICAL.lower()
  35. if reporting_state_warning in reporting and \
  36. 'count' in reporting[reporting_state_warning]:
  37. self.warning_count = reporting[reporting_state_warning]['count']
  38. if reporting_state_critical in reporting and \
  39. 'count' in reporting[reporting_state_critical]:
  40. self.critical_count = reporting[reporting_state_critical]['count']
  41. if self.critical_count <= self.warning_count:
  42. if logger.isEnabledFor(logging.DEBUG):
  43. logger.debug("[Alert][{0}] The CRITICAL value of {1} must be greater than the WARNING value of {2}".format(
  44. self.get_name(), self.critical_count, self.warning_count))
  45. def _collect(self):
  46. component = UNKNOWN_COMPONENT
  47. if 'componentName' in self.alert_meta:
  48. component = self.alert_meta['componentName']
  49. if logger.isEnabledFor(logging.DEBUG):
  50. logger.debug("[Alert][{0}] Checking recovery operations for {1}".format(
  51. self.get_name(), component))
  52. recovery_action_info = {}
  53. recovery_actions = self.recovery_manager.get_actions_copy()
  54. if component in recovery_actions:
  55. recovery_action_info = recovery_actions[component]
  56. warned_threshold_reached = False
  57. if 'warnedThresholdReached' in recovery_action_info:
  58. warned_threshold_reached = recovery_action_info['warnedThresholdReached']
  59. recovered_times = 0
  60. lastResetText = ""
  61. # The alert should not go away if warned_threshold_reached (max_lifetime_count reached)
  62. if not self.recovery_manager.is_action_info_stale(component) or warned_threshold_reached:
  63. if 'count' in recovery_action_info:
  64. recovered_times = recovery_action_info['count']
  65. if 'lastReset' in recovery_action_info:
  66. lastResetText = " since " + str(datetime.datetime.fromtimestamp(recovery_action_info['lastReset']))
  67. if recovered_times >= self.critical_count or warned_threshold_reached:
  68. result = self.RESULT_CRITICAL
  69. elif recovered_times >= self.warning_count:
  70. result = self.RESULT_WARNING
  71. elif recovered_times < self.warning_count and \
  72. recovered_times < self.critical_count:
  73. result = self.RESULT_OK
  74. else:
  75. result = self.RESULT_UNKNOWN
  76. return (result, [lastResetText, recovered_times, component])
  77. def _get_reporting_text(self, state):
  78. '''
  79. Gets the default reporting text to use when the alert definition does not
  80. contain any.
  81. :param state: the state of the alert in uppercase (such as OK, WARNING, etc)
  82. :return: the parametrized text
  83. '''
  84. if state == self.RESULT_OK:
  85. return 'No recovery operations executed for {2}{0}.'
  86. return '{1} recovery operations executed for {2}{0}.'