run-queue

#!/usr/bin/env python3

# This file is part of Cockpit.
#
# Copyright (C) 2018 Red Hat, Inc.
#
# Cockpit is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
#
# Cockpit is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Cockpit; If not, see <http://www.gnu.org/licenses/>.

import argparse
import json
import logging
import os
import random
import smtplib
import socket
import subprocess
import sys
import time

import pika

from lib.directories import get_images_data_dir
from lib.network import redhat_network
from lib.stores import LOG_STORE
from task import distributed_queue

logging.basicConfig(level=logging.INFO)

statistics_queue = os.environ.get("RUN_STATISTICS_QUEUE")

# Returns a command to execute and the delivery tag needed to ack the message


def consume_webhook_queue(channel, amqp):
    # interpret payload
    # call tests-scan or issue-scan appropriately
    method_frame, header_frame, body = channel.basic_get(queue='webhook')
    if not method_frame:
        return None, None

    body = json.loads(body)
    event = body['event']
    request = body['request']
    repo = None
    cmd = None
    if event == 'pull_request':
        pull_request = request['pull_request']
        repo = pull_request['base']['repo']['full_name']
        action = request['action']
        # scan for body changes (edited) and the bots label; we only use that for image refreshes in the bots repo
        if repo.endswith('/bots') and (
                action == 'labeled' or (action == 'edited' and 'body' in request.get('changes', {}))):
            cmd = ['./issue-scan', '--issues-data', json.dumps(request), '--amqp', amqp]
        elif action in ['opened', 'synchronize']:
            cmd = ['./tests-scan', '--pull-data', json.dumps(request), '--amqp', amqp, '--repo', repo]
        # When PR was merged, generate task for storing tests
        elif action == 'closed' and pull_request.get('merged', False):
            sha = pull_request['head']['sha']
            db = os.path.join(get_images_data_dir(), "test-results.db")
            cmd = (f"./store-tests --db {db} --repo {repo} {sha} && "
                   f"./prometheus-stats --db {db} --s3 {os.path.join(LOG_STORE, 'prometheus')}")
            body = {
                "command": cmd,
            }
            channel.basic_publish('', 'statistics', json.dumps(body),
                                  properties=pika.BasicProperties(priority=distributed_queue.MAX_PRIORITY))
            cmd = None

    elif event == 'status':
        repo = request['repository']['full_name']
        sha = request['sha']
        context = request['context']
        cmd = ['./tests-scan', '--sha', sha, '--amqp', amqp, '--context', context, '--repo', repo]
    elif event == 'issues':
        action = request['action']
        # scan for opened, body changes (edited), and the bots label
        if action in ['opened', 'labeled'] or (action == 'edited' and 'body' in request.get('changes', {})):
            cmd = ['./issue-scan', '--issues-data', json.dumps(request), '--amqp', amqp]
    else:
        logging.error('Unkown event type in the webhook queue')
        return None, None

    return cmd, method_frame.delivery_tag

# Returns a command to execute and the delivery tag needed to ack the message


def consume_task_queue(channel, amqp, declare_public_result, declare_rhel_result, declare_stats_result):
    if statistics_queue and declare_stats_result.method.message_count > 0:
        # statistics queue is quick to process, always do that first
        queue = 'statistics'
    elif os.path.exists('/dev/kvm') or os.getenv('COCKPIT_TESTMAP_INJECT'):
        # only process test queues in capable environments: with KVM support, or during integration tests
        queue = 'public'
        if redhat_network():
            # Try the rhel queue if the public queue is empty
            if declare_public_result.method.message_count == 0:
                queue = 'rhel'
                # If both are non-empty, shuffle
            elif declare_rhel_result.method.message_count > 0:
                queue = ['public', 'rhel'][random.randrange(2)]
    else:
        # nothing to do
        return None, None

    method_frame, header_frame, body = channel.basic_get(queue=queue)
    if not method_frame:
        return None, None

    body = json.loads(body)
    if job := body.get('job'):
        command = ['./job-runner', 'json', json.dumps(job)]
    else:
        command = body['command']
    return command, method_frame.delivery_tag


def mail_notification(body):
    mx = os.environ.get("TEST_NOTIFICATION_MX")
    if not mx:
        return

    sender = "noreply@redhat.com"
    receivers = os.environ.get("TEST_NOTIFICATION_TO", "").split(',')
    # if sending the notification fails, then fail the pod -- we do *not* want to ack messages in this case,
    # otherwise we would silently miss failure notifications; thus no exception handling here
    s = smtplib.SMTP(mx)
    s.sendmail(sender, receivers, """From: %s
To: %s
Subject: cockpituous: run-queue crash on %s

%s""" % (sender, ', '.join(receivers), socket.gethostname(), body))


# Consume from the webhook queue and republish to the task queue via *-scan
# Consume from the task queue (endpoint)
def main():
    parser = argparse.ArgumentParser(description='Bot: read a single test command from the queue and execute it')
    parser.add_argument('--amqp', default=distributed_queue.DEFAULT_AMQP_SERVER,
                        help='The host:port of the AMQP server to consume from (default: %(default)s)')
    opts = parser.parse_args()

    with distributed_queue.DistributedQueue(opts.amqp, ['webhook', 'rhel', 'public', 'statistics']) as q:
        channel = q.channel

        cmd, delivery_tag = consume_webhook_queue(channel, opts.amqp)
        if not cmd and delivery_tag:
            logging.info("Webhook message interpretation generated no command")
            channel.basic_ack(delivery_tag)
            return 0

        if not cmd:
            cmd, delivery_tag = consume_task_queue(
                channel,
                opts.amqp,
                q.declare_results['public'],
                q.declare_results['rhel'],
                q.declare_results['statistics'])
        if not cmd:
            logging.info("All queues are empty")
            return 1

        if isinstance(cmd, list):
            cmd_str = ' '.join(cmd)
        else:
            cmd_str = cmd
        logging.info("Consuming message with command: %s", cmd_str)
        p = subprocess.Popen(cmd, shell=isinstance(cmd, str))
        while p.poll() is None:
            q.connection.process_data_events()
            time.sleep(3)
        if p.returncode != 0:
            logging.error("%s failed with exit code %i", cmd_str, p.returncode)
            mail_notification("""The queue command

  %s

failed with exit code %i. Please check the container logs for details.""" % (cmd_str, p.returncode))

        channel.basic_ack(delivery_tag)

    return 0


if __name__ == '__main__':
    sys.exit(main())