# Copyright (c) 2024 Rackslab
#
# This file is part of Slurm-web.
#
# SPDX-License-Identifier: MIT


from unittest import mock
import random

from ClusterShell.NodeSet import NodeSet

from slurmweb.version import get_version
from slurmweb.slurmrestd.errors import (
    SlurmrestConnectionError,
    SlurmrestdInvalidResponseError,
)
from slurmweb.cache import CachingService
from slurmweb.views.agent import racksdb_get_version

from slurmweb.slurmrestd import TERMINAL_JOB_STATES

from ..lib.agent import TestAgentBase, ModifyActionsInPolicy
from ..lib.utils import all_slurm_api_versions, flask_404_description


class TestAgentViews(TestAgentBase):
    def setUp(self):
        self.setup_client()

    #
    # Generic routes (without slurmrestd requests)
    #

    def test_version(self):
        response = self.client.get("/version")
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.text, f"Slurm-web agent v{get_version()}\n")

    def test_info(self):
        response = self.client.get("/info")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, dict)
        self.assertEqual(len(response.json.keys()), 6)
        self.assertIn("cluster", response.json)
        self.assertEqual(response.json["cluster"], "test")
        self.assertIn("racksdb", response.json)
        self.assertIsInstance(response.json["racksdb"], dict)
        self.assertIn("enabled", response.json["racksdb"])
        self.assertIsInstance(response.json["racksdb"]["enabled"], bool)
        self.assertEqual(response.json["racksdb"]["enabled"], self.app.racksdb_active)
        self.assertIn("version", response.json["racksdb"])
        self.assertEqual(response.json["racksdb"]["version"], racksdb_get_version())
        self.assertIn("infrastructure", response.json["racksdb"])
        self.assertEqual(response.json["racksdb"]["infrastructure"], "test")
        self.assertIn("metrics", response.json)
        self.assertIsInstance(response.json["metrics"], bool)
        self.assertIn("cache", response.json)
        self.assertIsInstance(response.json["cache"], bool)
        self.assertIn("version", response.json)
        self.assertIsInstance(response.json["version"], str)
        self.assertEqual(response.json["version"], get_version())
        self.assertIn("slurmdbd", response.json)
        self.assertEqual(response.json["slurmdbd"]["jobs_max_hours"], 168)

    #
    # General error cases
    #

    def test_request_slurmrestd_connection_error(self):
        self.app.slurmrestd._request = mock.Mock(
            side_effect=SlurmrestConnectionError("connection error")
        )
        response = self.client.get(f"/v{get_version()}/jobs")
        self.assertEqual(response.status_code, 500)
        self.assertEqual(
            response.json,
            {
                "code": 500,
                "description": "Unable to connect to slurmrestd: connection error",
                "name": "Internal Server Error",
            },
        )

    def test_request_slurmrestd_invalid_type(self):
        self.app.slurmrestd._request = mock.Mock(
            side_effect=SlurmrestdInvalidResponseError("invalid type")
        )
        response = self.client.get(f"/v{get_version()}/jobs")
        self.assertEqual(response.status_code, 500)
        self.assertEqual(
            response.json,
            {
                "code": 500,
                "description": "Invalid response from slurmrestd: invalid type",
                "name": "Internal Server Error",
            },
        )

    @all_slurm_api_versions
    def test_request_slurmrestd_not_found(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [slurm_not_found_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-not-found", None)],
        )

        response = self.client.get(f"/v{get_version()}/jobs")
        self.assertEqual(response.status_code, 404)
        self.assertEqual(
            response.json,
            {
                "code": 404,
                "description": "URL not found on slurmrestd: /mocked/query",
                "name": "Not Found",
            },
        )

    @all_slurm_api_versions
    def test_request_slurmrestd_authentication_error(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [slurm_not_found_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-jwt-invalid-headers", None)],
        )
        response = self.client.get(f"/v{get_version()}/jobs")
        # FIXME: There is a regression in Slurm 25.11.0 which return HTTP/500 in this
        # case, see https://support.schedmd.com/show_bug.cgi?id=24052 for details.
        # This is a temporary workaround to accept both HTTP/401 and HTTP/500.
        self.assertIn(response.status_code, [401, 500])
        if response.status_code == 401:
            self.assertEqual(
                response.json,
                {
                    "code": 401,
                    "description": "Authentication error on slurmrestd: /mocked/query",
                    "name": "Unauthorized",
                },
            )
        else:
            self.assertEqual(
                response.json,
                {
                    "code": 500,
                    "description": "Authentication error on slurmrestd: /mocked/query",
                    "name": "Internal Server Error",
                },
            )

    def test_request_agent_not_found(self):
        response = self.client.get("/fail")
        self.assertEqual(response.status_code, 404)
        self.assertEqual(
            response.json,
            {
                "code": 404,
                "description": flask_404_description,
                "name": "Not Found",
            },
        )

    #
    # slurmrestd ressources
    #

    @all_slurm_api_versions
    def test_request_jobs(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [jobs_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-jobs", "jobs")],
        )

        response = self.client.get(f"/v{get_version()}/jobs")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, list)
        active_jobs_asset = [
            job
            for job in jobs_asset
            if not any(state in TERMINAL_JOB_STATES for state in job["job_state"])
        ]
        self.assertGreater(
            len(jobs_asset) - len(active_jobs_asset),
            0,
            "fixture must include terminal jobs to verify exclusion",
        )
        self.assertEqual(len(response.json), len(active_jobs_asset))
        for idx in range(len(response.json)):
            self.assertEqual(
                response.json[idx]["job_id"], active_jobs_asset[idx]["job_id"]
            )
            self.assertFalse(
                any(
                    state in TERMINAL_JOB_STATES
                    for state in response.json[idx]["job_state"]
                )
            )

    def test_request_jobs_own_only(self):
        with ModifyActionsInPolicy(
            self.app.policy, "user", remove="jobs-view", add="jobs-view-own"
        ):
            with mock.patch.object(
                self.app.slurmrestd,
                "jobs_current",
                return_value=[
                    {
                        "job_id": 1,
                        "user_name": "test",
                        "job_state": ["RUNNING"],
                        "nodes": "node1",
                    },
                    {
                        "job_id": 2,
                        "user_name": "other",
                        "job_state": ["PENDING"],
                        "nodes": "node2",
                    },
                ],
            ):
                response = self.client.get(f"/v{get_version()}/jobs")
        self.assertEqual(response.status_code, 200)
        self.assertEqual(len(response.json), 1)
        self.assertEqual(response.json[0]["job_id"], 1)

    def test_request_jobs_own_precedence(self):
        # Add jobs-view-own permission to user role in addition to jobs-view permission.
        # In this case, jobs-view should take precedence and user should see all jobs.
        with ModifyActionsInPolicy(self.app.policy, "user", add="jobs-view-own"):
            with mock.patch.object(
                self.app.slurmrestd,
                "jobs_current",
                return_value=[
                    {
                        "job_id": 1,
                        "user_name": "test",
                        "job_state": ["RUNNING"],
                        "nodes": "node1",
                    },
                    {
                        "job_id": 2,
                        "user_name": "other",
                        "job_state": ["PENDING"],
                        "nodes": "node2",
                    },
                ],
            ):
                response = self.client.get(f"/v{get_version()}/jobs")
        self.assertEqual(len(response.json), 2)

    def test_request_jobs_denied(self):
        with ModifyActionsInPolicy(self.app.policy, "user", remove="jobs-view"):
            response = self.client.get(f"/v{get_version()}/jobs")
        self.assertEqual(response.status_code, 403)
        self.assertEqual(
            response.json,
            {
                "code": 403,
                "description": (
                    "User is not allowed to perform action jobs-view or jobs-view-own"
                ),
                "name": "Forbidden",
            },
        )

    @all_slurm_api_versions
    def test_request_jobs_node(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [jobs_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-jobs", "jobs")],
        )

        def terminated(job):
            return any(
                state in TERMINAL_JOB_STATES for state in job.get("job_state", [])
            )

        # Select random busy node on cluster
        busy_nodes = NodeSet()
        for job in jobs_asset:
            if not terminated(job) and job["nodes"]:
                busy_nodes.update(job["nodes"])
        if not len(busy_nodes):
            self.skipTest("No current jobs with allocated nodes in test asset")
        random_busy_node = random.choice(busy_nodes)

        response = self.client.get(f"/v{get_version()}/jobs?node={random_busy_node}")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, list)

        # Check we have results but less that full list of jobs.
        self.assertNotEqual(len(response.json), 0)
        self.assertLess(len(response.json), len(jobs_asset))

        # Check all jobs are not completed and have the random busy node allocated.
        for job in response.json:
            self.assertFalse(
                any(state in TERMINAL_JOB_STATES for state in job.get("job_state", []))
            )
            self.assertIn(
                random_busy_node,
                NodeSet(job["nodes"]),
            )

    @all_slurm_api_versions
    def test_request_jobs_past(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurmdb-jobs", "jobs")],
        )
        response = self.client.get(f"/v{get_version()}/jobs/past?hours=6")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, list)
        self.assertGreater(len(response.json), 0)
        for job in response.json:
            for state in job.get("state", {}).get("current", []):
                self.assertIn(state, TERMINAL_JOB_STATES)

    def test_request_jobs_past_missing_hours(self):
        self.setup_client()
        response = self.client.get(f"/v{get_version()}/jobs/past")
        self.assertEqual(response.status_code, 400)

    def test_request_jobs_past_own_only(self):
        with ModifyActionsInPolicy(
            self.app.policy,
            "user",
            remove="jobs-view-past",
            add="jobs-view-past-own",
        ):
            with mock.patch.object(
                self.app.slurmrestd,
                "jobs_past",
                return_value=[
                    {"job_id": 10, "user": "test", "state": {"current": ["COMPLETED"]}},
                    {"job_id": 11, "user": "other", "state": {"current": ["FAILED"]}},
                ],
            ) as jobs_past:
                response = self.client.get(f"/v{get_version()}/jobs/past?hours=6")
        self.assertEqual(response.status_code, 200)
        self.assertEqual(len(response.json), 1)
        self.assertEqual(response.json[0]["job_id"], 10)
        jobs_past.assert_called_once_with(6)

    def test_request_jobs_past_own_precedence(self):
        # Add jobs-view-past-own in addition to jobs-view-past: broad action wins.
        with ModifyActionsInPolicy(self.app.policy, "user", add="jobs-view-past-own"):
            with mock.patch.object(
                self.app.slurmrestd,
                "jobs_past",
                return_value=[
                    {"job_id": 10, "user": "test", "state": {"current": ["COMPLETED"]}},
                    {"job_id": 11, "user": "other", "state": {"current": ["FAILED"]}},
                ],
            ):
                response = self.client.get(f"/v{get_version()}/jobs/past?hours=6")
        self.assertEqual(len(response.json), 2)

    def test_request_jobs_past_denied(self):
        with ModifyActionsInPolicy(
            self.app.policy,
            "user",
            remove=("jobs-view-past", "jobs-view-past-own"),
        ):
            with mock.patch.object(self.app.slurmrestd, "jobs_past") as jobs_past:
                response = self.client.get(f"/v{get_version()}/jobs/past?hours=6")
        jobs_past.assert_not_called()
        self.assertEqual(response.status_code, 403)
        self.assertEqual(
            response.json,
            {
                "code": 403,
                "description": (
                    "User is not allowed to perform action "
                    "jobs-view-past or jobs-view-past-own"
                ),
                "name": "Forbidden",
            },
        )

    @all_slurm_api_versions
    def test_request_job_running(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [slurmdb_job_asset, slurm_job_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurmdb-job-running", "jobs"), ("slurm-job-running", "jobs")],
        )
        response = self.client.get(f"/v{get_version()}/job/1")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, dict)
        self.assertEqual(response.json["state"], slurmdb_job_asset[0]["state"])
        self.assertEqual(
            response.json["association"], slurmdb_job_asset[0]["association"]
        )
        self.assertEqual(
            response.json["tres_req_str"], slurm_job_asset[0]["tres_req_str"]
        )

    @all_slurm_api_versions
    def test_request_job_pending(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [slurmdb_job_asset, slurm_job_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurmdb-job-pending", "jobs"), ("slurm-job-pending", "jobs")],
        )
        response = self.client.get(f"/v{get_version()}/job/1")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, dict)
        self.assertEqual(response.json["state"], slurmdb_job_asset[0]["state"])
        self.assertEqual(
            response.json["association"], slurmdb_job_asset[0]["association"]
        )
        self.assertEqual(
            response.json["tres_req_str"], slurm_job_asset[0]["tres_req_str"]
        )

    @all_slurm_api_versions
    def test_request_job_completed(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [slurmdb_job_asset, slurm_job_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurmdb-job-completed", "jobs"), ("slurm-job-completed", "jobs")],
        )
        response = self.client.get(f"/v{get_version()}/job/1")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, dict)
        self.assertEqual(response.json["state"], slurmdb_job_asset[0]["state"])
        self.assertEqual(
            response.json["association"], slurmdb_job_asset[0]["association"]
        )
        self.assertEqual(
            response.json["tres_req_str"], slurm_job_asset[0]["tres_req_str"]
        )

    @all_slurm_api_versions
    def test_request_job_failed(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [slurmdb_job_asset, slurm_job_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurmdb-job-failed", "jobs"), ("slurm-job-failed", "jobs")],
        )
        response = self.client.get(f"/v{get_version()}/job/1")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, dict)
        self.assertEqual(response.json["state"], slurmdb_job_asset[0]["state"])
        self.assertEqual(
            response.json["association"], slurmdb_job_asset[0]["association"]
        )
        self.assertEqual(
            response.json["tres_req_str"], slurm_job_asset[0]["tres_req_str"]
        )

    @all_slurm_api_versions
    def test_request_job_timeout(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [slurmdb_job_asset, slurm_job_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurmdb-job-timeout", "jobs"), ("slurm-job-timeout", "jobs")],
        )
        response = self.client.get(f"/v{get_version()}/job/1")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, dict)
        self.assertEqual(response.json["state"], slurmdb_job_asset[0]["state"])
        self.assertEqual(
            response.json["association"], slurmdb_job_asset[0]["association"]
        )
        self.assertEqual(
            response.json["tres_req_str"], slurm_job_asset[0]["tres_req_str"]
        )

    @all_slurm_api_versions
    def test_request_job_archived(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [slurmdb_job_asset, slurm_job_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurmdb-job-archived", "jobs"), ("slurm-job-archived", None)],
        )
        response = self.client.get(f"/v{get_version()}/job/1")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, dict)
        self.assertEqual(response.json["state"], slurmdb_job_asset[0]["state"])
        self.assertEqual(
            response.json["association"], slurmdb_job_asset[0]["association"]
        )
        # Unable to retrieve tres_req_str only provided by slurmctld
        self.assertNotIn("tres_req_str", response.json)

    @all_slurm_api_versions
    def test_request_job_not_found(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [slurmdb_job_asset, slurm_job_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurmdb-job-unfound", "jobs"), ("slurm-job-unfound", None)],
        )
        response = self.client.get(f"/v{get_version()}/job/1")
        self.assertEqual(response.status_code, 404)
        self.assertEqual(
            response.json,
            {
                "code": 404,
                "description": "URL not found on slurmrestd: Job 1 not found",
                "name": "Not Found",
            },
        )

    def test_request_job_own_only(self):
        with ModifyActionsInPolicy(
            self.app.policy, "user", remove="jobs-view", add="jobs-view-own"
        ):
            with mock.patch.object(
                self.app.slurmrestd,
                "job",
                return_value={"job_id": 1, "user": "test"},
            ):
                response = self.client.get(f"/v{get_version()}/job/1")
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.json, {"job_id": 1, "user": "test"})

    def test_request_job_own_only_not_found(self):
        with ModifyActionsInPolicy(
            self.app.policy, "user", remove="jobs-view", add="jobs-view-own"
        ):
            with mock.patch.object(
                self.app.slurmrestd,
                "job",
                return_value={"job_id": 2, "user": "other"},
            ):
                response = self.client.get(f"/v{get_version()}/job/2")
        self.assertEqual(response.status_code, 404)
        self.assertEqual(
            response.json,
            {
                "code": 404,
                "description": "Job not found",
                "name": "Not Found",
            },
        )

    def test_request_job_own_precedence(self):
        # Add jobs-view-own in addition to jobs-view: broad action wins.
        other_job = {"job_id": 2, "user": "other"}
        with ModifyActionsInPolicy(self.app.policy, "user", add="jobs-view-own"):
            with mock.patch.object(self.app.slurmrestd, "job", return_value=other_job):
                response = self.client.get(f"/v{get_version()}/job/2")
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.json, other_job)

    def test_request_job_denied(self):
        with ModifyActionsInPolicy(
            self.app.policy, "user", remove=("jobs-view", "jobs-view-own")
        ):
            with mock.patch.object(self.app.slurmrestd, "job") as job_mock:
                response = self.client.get(f"/v{get_version()}/job/1")
        job_mock.assert_not_called()
        self.assertEqual(response.status_code, 403)
        self.assertEqual(
            response.json,
            {
                "code": 403,
                "description": (
                    "User is not allowed to perform action jobs-view or jobs-view-own"
                ),
                "name": "Forbidden",
            },
        )

    @all_slurm_api_versions
    def test_request_nodes(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [nodes_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-nodes", "nodes")],
        )
        response = self.client.get(f"/v{get_version()}/nodes")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, list)
        self.assertEqual(len(response.json), len(nodes_asset))
        for idx in range(len(response.json)):
            self.assertEqual(response.json[idx]["name"], nodes_asset[idx]["name"])

    @all_slurm_api_versions
    def test_request_node_idle(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [node_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-node-idle", "nodes")],
        )
        response = self.client.get(f"/v{get_version()}/node/cn01")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, dict)
        self.assertEqual(response.json["name"], node_asset[0]["name"])
        self.assertIn("IDLE", response.json["state"])

    @all_slurm_api_versions
    def test_request_node_allocated(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [node_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-node-allocated", "nodes")],
        )
        response = self.client.get(f"/v{get_version()}/node/cn01")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, dict)
        self.assertEqual(response.json["name"], node_asset[0]["name"])
        self.assertIn("ALLOCATED", response.json["state"])

    @all_slurm_api_versions
    def test_request_node_mixed(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [node_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-node-mixed", "nodes")],
        )
        response = self.client.get(f"/v{get_version()}/node/cn01")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, dict)
        self.assertEqual(response.json["name"], node_asset[0]["name"])
        self.assertIn("MIXED", response.json["state"])

    @all_slurm_api_versions
    def test_request_node_down(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [node_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-node-down", "nodes")],
        )
        response = self.client.get(f"/v{get_version()}/node/cn01")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, dict)
        self.assertEqual(response.json["name"], node_asset[0]["name"])
        self.assertIn("DOWN", response.json["state"])

    @all_slurm_api_versions
    def test_request_node_drained(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [node_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-node-drain", "nodes")],
        )
        response = self.client.get(f"/v{get_version()}/node/cn01")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, dict)
        self.assertEqual(response.json["name"], node_asset[0]["name"])
        self.assertIn("DRAIN", response.json["state"])

    @all_slurm_api_versions
    def test_request_node_draining(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [node_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-node-draining", "nodes")],
        )
        response = self.client.get(f"/v{get_version()}/node/cn01")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, dict)
        self.assertEqual(response.json["name"], node_asset[0]["name"])
        self.assertIn("DRAIN", response.json["state"])
        self.assertTrue(
            "MIXED" in response.json["state"] or "ALLOCATED" in response.json["state"]
        )

    @all_slurm_api_versions
    def test_request_node_not_found(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-node-unfound", None)],
        )
        response = self.client.get(f"/v{get_version()}/node/not-found")
        self.assertEqual(response.status_code, 404)
        self.assertEqual(
            response.json,
            {
                "code": 404,
                "description": "URL not found on slurmrestd: Node not-found not found",
                "name": "Not Found",
            },
        )

    @all_slurm_api_versions
    def test_request_stats(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [jobs_asset, nodes_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [
                ("slurm-jobs", "jobs"),
                ("slurm-nodes", "nodes"),
            ],
        )
        response = self.client.get(f"/v{get_version()}/stats")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, dict)
        self.assertIn("jobs", response.json)
        self.assertIn("resources", response.json)
        self.assertEqual(response.json["jobs"]["total"], len(jobs_asset))
        self.assertEqual(
            response.json["jobs"]["running"],
            len([job for job in jobs_asset if "RUNNING" in job["job_state"]]),
        )
        self.assertCountEqual(
            response.json["resources"].keys(), ["nodes", "cores", "memory", "gpus"]
        )
        self.assertEqual(response.json["resources"]["nodes"], len(nodes_asset))
        self.assertEqual(
            response.json["resources"]["cores"],
            sum([node["cpus"] for node in nodes_asset]),
        )
        self.assertEqual(
            response.json["resources"]["memory"],
            sum([node["real_memory"] for node in nodes_asset]),
        )
        self.assertEqual(
            response.json["resources"]["gpus"],
            sum(
                [
                    self.app.slurmrestd.node_gres_extract_gpus(node["gres"])
                    for node in nodes_asset
                ]
            ),
        )

    @all_slurm_api_versions
    def test_request_partitions(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [partitions_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-partitions", "partitions")],
        )
        response = self.client.get(f"/v{get_version()}/partitions")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, list)
        self.assertEqual(len(response.json), len(partitions_asset))
        for idx in range(len(response.json)):
            self.assertEqual(response.json[idx]["name"], partitions_asset[idx]["name"])

    @all_slurm_api_versions
    def test_request_qos(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [qos_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-qos", "qos")],
        )
        response = self.client.get(f"/v{get_version()}/qos")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, list)
        self.assertEqual(len(response.json), len(qos_asset))
        for idx in range(len(response.json)):
            self.assertEqual(response.json[idx]["name"], qos_asset[idx]["name"])

    @all_slurm_api_versions
    def test_request_reservations(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [reservations_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-reservations", "reservations")],
        )
        response = self.client.get(f"/v{get_version()}/reservations")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, list)
        self.assertEqual(len(response.json), len(reservations_asset))
        for idx in range(len(response.json)):
            self.assertEqual(
                response.json[idx]["name"], reservations_asset[idx]["name"]
            )

    @all_slurm_api_versions
    def test_request_accounts(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [accounts_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurm-accounts", "accounts")],
        )
        response = self.client.get(f"/v{get_version()}/accounts")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, list)
        self.assertEqual(len(response.json), len(accounts_asset))
        for idx in range(len(response.json)):
            self.assertEqual(response.json[idx]["name"], accounts_asset[idx]["name"])

    @all_slurm_api_versions
    def test_request_associations(self, slurm_version, api_version):
        self.setup_slurmrestd(slurm_version, api_version)
        [associations_asset] = self.mock_slurmrestd_responses(
            slurm_version,
            api_version,
            [("slurmdb-associations", "associations")],
        )
        response = self.client.get(f"/v{get_version()}/associations")
        self.assertEqual(response.status_code, 200)
        self.assertIsInstance(response.json, list)
        self.assertEqual(len(response.json), len(associations_asset))
        for idx in range(len(response.json)):
            self.assertEqual(
                response.json[idx]["account"], associations_asset[idx]["account"]
            )
            self.assertEqual(
                response.json[idx]["user"], associations_asset[idx]["user"]
            )

    def test_cache_stats_disabled(self):
        with self.assertLogs("slurmweb", level="WARNING") as cm:
            response = self.client.get(f"/v{get_version()}/cache/stats")
        self.assertEqual(response.status_code, 501)
        self.assertEqual(
            response.json,
            {
                "code": 501,
                "description": (
                    "Cache service is disabled, unable to query cache statistics"
                ),
                "name": "Not Implemented",
            },
        )
        self.assertEqual(
            cm.output,
            [
                "WARNING:slurmweb.views.agent:Cache service is disabled, unable to "
                "query cache statistics"
            ],
        )

    def test_cache_stats(self):
        self.app.cache = mock.Mock(spec=CachingService)
        self.app.cache.metrics.return_value = (
            {"jobs": 10, "nodes": 5},
            {"jobs": 8, "nodes": 3},
            15,
            11,
        )
        response = self.client.get(f"/v{get_version()}/cache/stats")
        self.assertEqual(response.status_code, 200)
        self.assertEqual(
            response.json,
            {
                "hit": {"keys": {"jobs": 10, "nodes": 5}, "total": 15},
                "miss": {"keys": {"jobs": 8, "nodes": 3}, "total": 11},
            },
        )

    def test_cache_reset_disabled(self):
        with self.assertLogs("slurmweb", level="WARNING") as cm:
            response = self.client.post(f"/v{get_version()}/cache/reset")
        self.assertEqual(response.status_code, 501)
        self.assertEqual(
            response.json,
            {
                "code": 501,
                "description": ("Cache service is disabled, unable to reset cache"),
                "name": "Not Implemented",
            },
        )
        self.assertEqual(
            cm.output,
            [
                "WARNING:slurmweb.views.agent:Cache service is disabled, unable to "
                "reset cache"
            ],
        )

    def test_cache_reset(self):
        self.app.cache = mock.Mock(spec=CachingService)
        self.app.cache.metrics.return_value = (
            {"jobs": 0},
            {"jobs": 0},
            0,
            0,
        )
        response = self.client.post(f"/v{get_version()}/cache/reset")
        self.assertEqual(response.status_code, 200)
        # Ensure reset() was called before metrics()
        self.app.cache.reset.assert_called_once()
        self.app.cache.metrics.assert_called_once()
        self.assertEqual(
            response.json,
            {
                "hit": {"keys": {"jobs": 0}, "total": 0},
                "miss": {"keys": {"jobs": 0}, "total": 0},
            },
        )

    def test_request_metrics(self):
        # Metrics feature is disabled in this test case, check that the corresponding
        # endpoint returns HTTP/404 (not found).
        response = self.client.get("/metrics")
        self.assertEqual(response.status_code, 404)
