qemu-e2k/util/interval-tree.c

/* SPDX-License-Identifier: GPL-2.0-or-later */

#include "qemu/osdep.h"
#include "qemu/interval-tree.h"
#include "qemu/atomic.h"

/*
 * Red Black Trees.
 *
 * For now, don't expose Linux Red-Black Trees separately, but retain the
 * separate type definitions to keep the implementation sane, and allow
 * the possibility of separating them later.
 *
 * Derived from include/linux/rbtree_augmented.h and its dependencies.
 */

/*
 * red-black trees properties:  https://en.wikipedia.org/wiki/Rbtree
 *
 *  1) A node is either red or black
 *  2) The root is black
 *  3) All leaves (NULL) are black
 *  4) Both children of every red node are black
 *  5) Every simple path from root to leaves contains the same number
 *     of black nodes.
 *
 *  4 and 5 give the O(log n) guarantee, since 4 implies you cannot have two
 *  consecutive red nodes in a path and every red node is therefore followed by
 *  a black. So if B is the number of black nodes on every simple path (as per
 *  5), then the longest possible path due to 4 is 2B.
 *
 *  We shall indicate color with case, where black nodes are uppercase and red
 *  nodes will be lowercase. Unknown color nodes shall be drawn as red within
 *  parentheses and have some accompanying text comment.
 *
 * Notes on lockless lookups:
 *
 * All stores to the tree structure (rb_left and rb_right) must be done using
 * WRITE_ONCE [qatomic_set for QEMU]. And we must not inadvertently cause
 * (temporary) loops in the tree structure as seen in program order.
 *
 * These two requirements will allow lockless iteration of the tree -- not
 * correct iteration mind you, tree rotations are not atomic so a lookup might
 * miss entire subtrees.
 *
 * But they do guarantee that any such traversal will only see valid elements
 * and that it will indeed complete -- does not get stuck in a loop.
 *
 * It also guarantees that if the lookup returns an element it is the 'correct'
 * one. But not returning an element does _NOT_ mean it's not present.
 */

typedef enum RBColor
{
    RB_RED,
    RB_BLACK,
} RBColor;

typedef struct RBAugmentCallbacks {
    void (*propagate)(RBNode *node, RBNode *stop);
    void (*copy)(RBNode *old, RBNode *new);
    void (*rotate)(RBNode *old, RBNode *new);
} RBAugmentCallbacks;

static inline uintptr_t rb_pc(const RBNode *n)
{
    return qatomic_read(&n->rb_parent_color);
}

static inline void rb_set_pc(RBNode *n, uintptr_t pc)
{
    qatomic_set(&n->rb_parent_color, pc);
}

static inline RBNode *pc_parent(uintptr_t pc)
{
    return (RBNode *)(pc & ~1);
}

static inline RBNode *rb_parent(const RBNode *n)
{
    return pc_parent(rb_pc(n));
}

static inline RBNode *rb_red_parent(const RBNode *n)
{
    return (RBNode *)rb_pc(n);
}

static inline RBColor pc_color(uintptr_t pc)
{
    return (RBColor)(pc & 1);
}

static inline bool pc_is_red(uintptr_t pc)
{
    return pc_color(pc) == RB_RED;
}

static inline bool pc_is_black(uintptr_t pc)
{
    return !pc_is_red(pc);
}

static inline RBColor rb_color(const RBNode *n)
{
    return pc_color(rb_pc(n));
}

static inline bool rb_is_red(const RBNode *n)
{
    return pc_is_red(rb_pc(n));
}

static inline bool rb_is_black(const RBNode *n)
{
    return pc_is_black(rb_pc(n));
}

static inline void rb_set_black(RBNode *n)
{
    rb_set_pc(n, rb_pc(n) | RB_BLACK);
}

static inline void rb_set_parent_color(RBNode *n, RBNode *p, RBColor color)
{
    rb_set_pc(n, (uintptr_t)p | color);
}

static inline void rb_set_parent(RBNode *n, RBNode *p)
{
    rb_set_parent_color(n, p, rb_color(n));
}

static inline void rb_link_node(RBNode *node, RBNode *parent, RBNode **rb_link)
{
    node->rb_parent_color = (uintptr_t)parent;
    node->rb_left = node->rb_right = NULL;

    /*
     * Ensure that node is initialized before insertion,
     * as viewed by a concurrent search.
     */
    qatomic_set_mb(rb_link, node);
}

static RBNode *rb_next(RBNode *node)
{
    RBNode *parent;

    /* OMIT: if empty node, return null. */

    /*
     * If we have a right-hand child, go down and then left as far as we can.
     */
    if (node->rb_right) {
        node = node->rb_right;
        while (node->rb_left) {
            node = node->rb_left;
        }
        return node;
    }

    /*
     * No right-hand children. Everything down and left is smaller than us,
     * so any 'next' node must be in the general direction of our parent.
     * Go up the tree; any time the ancestor is a right-hand child of its
     * parent, keep going up. First time it's a left-hand child of its
     * parent, said parent is our 'next' node.
     */
    while ((parent = rb_parent(node)) && node == parent->rb_right) {
        node = parent;
    }

    return parent;
}

static inline void rb_change_child(RBNode *old, RBNode *new,
                                   RBNode *parent, RBRoot *root)
{
    if (!parent) {
        qatomic_set(&root->rb_node, new);
    } else if (parent->rb_left == old) {
        qatomic_set(&parent->rb_left, new);
    } else {
        qatomic_set(&parent->rb_right, new);
    }
}

static inline void rb_rotate_set_parents(RBNode *old, RBNode *new,
                                         RBRoot *root, RBColor color)
{
    uintptr_t pc = rb_pc(old);
    RBNode *parent = pc_parent(pc);

    rb_set_pc(new, pc);
    rb_set_parent_color(old, new, color);
    rb_change_child(old, new, parent, root);
}

static void rb_insert_augmented(RBNode *node, RBRoot *root,
                                const RBAugmentCallbacks *augment)
{
    RBNode *parent = rb_red_parent(node), *gparent, *tmp;

    while (true) {
        /*
         * Loop invariant: node is red.
         */
        if (unlikely(!parent)) {
            /*
             * The inserted node is root. Either this is the first node, or
             * we recursed at Case 1 below and are no longer violating 4).
             */
            rb_set_parent_color(node, NULL, RB_BLACK);
            break;
        }

        /*
         * If there is a black parent, we are done.  Otherwise, take some
         * corrective action as, per 4), we don't want a red root or two
         * consecutive red nodes.
         */
        if (rb_is_black(parent)) {
            break;
        }

        gparent = rb_red_parent(parent);

        tmp = gparent->rb_right;
        if (parent != tmp) {    /* parent == gparent->rb_left */
            if (tmp && rb_is_red(tmp)) {
                /*
                 * Case 1 - node's uncle is red (color flips).
                 *
                 *       G            g
                 *      / \          / \
                 *     p   u  -->   P   U
                 *    /            /
                 *   n            n
                 *
                 * However, since g's parent might be red, and 4) does not
                 * allow this, we need to recurse at g.
                 */
                rb_set_parent_color(tmp, gparent, RB_BLACK);
                rb_set_parent_color(parent, gparent, RB_BLACK);
                node = gparent;
                parent = rb_parent(node);
                rb_set_parent_color(node, parent, RB_RED);
                continue;
            }

            tmp = parent->rb_right;
            if (node == tmp) {
                /*
                 * Case 2 - node's uncle is black and node is
                 * the parent's right child (left rotate at parent).
                 *
                 *      G             G
                 *     / \           / \
                 *    p   U  -->    n   U
                 *     \           /
                 *      n         p
                 *
                 * This still leaves us in violation of 4), the
                 * continuation into Case 3 will fix that.
                 */
                tmp = node->rb_left;
                qatomic_set(&parent->rb_right, tmp);
                qatomic_set(&node->rb_left, parent);
                if (tmp) {
                    rb_set_parent_color(tmp, parent, RB_BLACK);
                }
                rb_set_parent_color(parent, node, RB_RED);
                augment->rotate(parent, node);
                parent = node;
                tmp = node->rb_right;
            }

            /*
             * Case 3 - node's uncle is black and node is
             * the parent's left child (right rotate at gparent).
             *
             *        G           P
             *       / \         / \
             *      p   U  -->  n   g
             *     /                 \
             *    n                   U
             */
            qatomic_set(&gparent->rb_left, tmp); /* == parent->rb_right */
            qatomic_set(&parent->rb_right, gparent);
            if (tmp) {
                rb_set_parent_color(tmp, gparent, RB_BLACK);
            }
            rb_rotate_set_parents(gparent, parent, root, RB_RED);
            augment->rotate(gparent, parent);
            break;
        } else {
            tmp = gparent->rb_left;
            if (tmp && rb_is_red(tmp)) {
                /* Case 1 - color flips */
                rb_set_parent_color(tmp, gparent, RB_BLACK);
                rb_set_parent_color(parent, gparent, RB_BLACK);
                node = gparent;
                parent = rb_parent(node);
                rb_set_parent_color(node, parent, RB_RED);
                continue;
            }

            tmp = parent->rb_left;
            if (node == tmp) {
                /* Case 2 - right rotate at parent */
                tmp = node->rb_right;
                qatomic_set(&parent->rb_left, tmp);
                qatomic_set(&node->rb_right, parent);
                if (tmp) {
                    rb_set_parent_color(tmp, parent, RB_BLACK);
                }
                rb_set_parent_color(parent, node, RB_RED);
                augment->rotate(parent, node);
                parent = node;
                tmp = node->rb_left;
            }

            /* Case 3 - left rotate at gparent */
            qatomic_set(&gparent->rb_right, tmp); /* == parent->rb_left */
            qatomic_set(&parent->rb_left, gparent);
            if (tmp) {
                rb_set_parent_color(tmp, gparent, RB_BLACK);
            }
            rb_rotate_set_parents(gparent, parent, root, RB_RED);
            augment->rotate(gparent, parent);
            break;
        }
    }
}

static void rb_insert_augmented_cached(RBNode *node,
                                       RBRootLeftCached *root, bool newleft,
                                       const RBAugmentCallbacks *augment)
{
    if (newleft) {
        root->rb_leftmost = node;
    }
    rb_insert_augmented(node, &root->rb_root, augment);
}

static void rb_erase_color(RBNode *parent, RBRoot *root,
                           const RBAugmentCallbacks *augment)
{
    RBNode *node = NULL, *sibling, *tmp1, *tmp2;

    while (true) {
        /*
         * Loop invariants:
         * - node is black (or NULL on first iteration)
         * - node is not the root (parent is not NULL)
         * - All leaf paths going through parent and node have a
         *   black node count that is 1 lower than other leaf paths.
         */
        sibling = parent->rb_right;
        if (node != sibling) {  /* node == parent->rb_left */
            if (rb_is_red(sibling)) {
                /*
                 * Case 1 - left rotate at parent
                 *
                 *     P               S
                 *    / \             / \ 
                 *   N   s    -->    p   Sr
                 *      / \         / \ 
                 *     Sl  Sr      N   Sl
                 */
                tmp1 = sibling->rb_left;
                qatomic_set(&parent->rb_right, tmp1);
                qatomic_set(&sibling->rb_left, parent);
                rb_set_parent_color(tmp1, parent, RB_BLACK);
                rb_rotate_set_parents(parent, sibling, root, RB_RED);
                augment->rotate(parent, sibling);
                sibling = tmp1;
            }
            tmp1 = sibling->rb_right;
            if (!tmp1 || rb_is_black(tmp1)) {
                tmp2 = sibling->rb_left;
                if (!tmp2 || rb_is_black(tmp2)) {
                    /*
                     * Case 2 - sibling color flip
                     * (p could be either color here)
                     *
                     *    (p)           (p)
                     *    / \           / \ 
                     *   N   S    -->  N   s
                     *      / \           / \ 
                     *     Sl  Sr        Sl  Sr
                     *
                     * This leaves us violating 5) which
                     * can be fixed by flipping p to black
                     * if it was red, or by recursing at p.
                     * p is red when coming from Case 1.
                     */
                    rb_set_parent_color(sibling, parent, RB_RED);
                    if (rb_is_red(parent)) {
                        rb_set_black(parent);
                    } else {
                        node = parent;
                        parent = rb_parent(node);
                        if (parent) {
                            continue;
                        }
                    }
                    break;
                }
                /*
                 * Case 3 - right rotate at sibling
                 * (p could be either color here)
                 *
                 *   (p)           (p)
                 *   / \           / \
                 *  N   S    -->  N   sl
                 *     / \             \
                 *    sl  Sr            S
                 *                       \
                 *                        Sr
                 *
                 * Note: p might be red, and then bot
                 * p and sl are red after rotation (which
                 * breaks property 4). This is fixed in
                 * Case 4 (in rb_rotate_set_parents()
                 *         which set sl the color of p
                 *         and set p RB_BLACK)
                 *
                 *   (p)            (sl)
                 *   / \            /  \
                 *  N   sl   -->   P    S
                 *       \        /      \
                 *        S      N        Sr
                 *         \
                 *          Sr
                 */
                tmp1 = tmp2->rb_right;
                qatomic_set(&sibling->rb_left, tmp1);
                qatomic_set(&tmp2->rb_right, sibling);
                qatomic_set(&parent->rb_right, tmp2);
                if (tmp1) {
                    rb_set_parent_color(tmp1, sibling, RB_BLACK);
                }
                augment->rotate(sibling, tmp2);
                tmp1 = sibling;
                sibling = tmp2;
            }
            /*
             * Case 4 - left rotate at parent + color flips
             * (p and sl could be either color here.
             *  After rotation, p becomes black, s acquires
             *  p's color, and sl keeps its color)
             *
             *      (p)             (s)
             *      / \             / \
             *     N   S     -->   P   Sr
             *        / \         / \
             *      (sl) sr      N  (sl)
             */
            tmp2 = sibling->rb_left;
            qatomic_set(&parent->rb_right, tmp2);
            qatomic_set(&sibling->rb_left, parent);
            rb_set_parent_color(tmp1, sibling, RB_BLACK);
            if (tmp2) {
                rb_set_parent(tmp2, parent);
            }
            rb_rotate_set_parents(parent, sibling, root, RB_BLACK);
            augment->rotate(parent, sibling);
            break;
        } else {
            sibling = parent->rb_left;
            if (rb_is_red(sibling)) {
                /* Case 1 - right rotate at parent */
                tmp1 = sibling->rb_right;
                qatomic_set(&parent->rb_left, tmp1);
                qatomic_set(&sibling->rb_right, parent);
                rb_set_parent_color(tmp1, parent, RB_BLACK);
                rb_rotate_set_parents(parent, sibling, root, RB_RED);
                augment->rotate(parent, sibling);
                sibling = tmp1;
            }
            tmp1 = sibling->rb_left;
            if (!tmp1 || rb_is_black(tmp1)) {
                tmp2 = sibling->rb_right;
                if (!tmp2 || rb_is_black(tmp2)) {
                    /* Case 2 - sibling color flip */
                    rb_set_parent_color(sibling, parent, RB_RED);
                    if (rb_is_red(parent)) {
                        rb_set_black(parent);
                    } else {
                        node = parent;
                        parent = rb_parent(node);
                        if (parent) {
                            continue;
                        }
                    }
                    break;
                }
                /* Case 3 - left rotate at sibling */
                tmp1 = tmp2->rb_left;
                qatomic_set(&sibling->rb_right, tmp1);
                qatomic_set(&tmp2->rb_left, sibling);
                qatomic_set(&parent->rb_left, tmp2);
                if (tmp1) {
                    rb_set_parent_color(tmp1, sibling, RB_BLACK);
                }
                augment->rotate(sibling, tmp2);
                tmp1 = sibling;
                sibling = tmp2;
            }
            /* Case 4 - right rotate at parent + color flips */
            tmp2 = sibling->rb_right;
            qatomic_set(&parent->rb_left, tmp2);
            qatomic_set(&sibling->rb_right, parent);
            rb_set_parent_color(tmp1, sibling, RB_BLACK);
            if (tmp2) {
                rb_set_parent(tmp2, parent);
            }
            rb_rotate_set_parents(parent, sibling, root, RB_BLACK);
            augment->rotate(parent, sibling);
            break;
        }
    }
}

static void rb_erase_augmented(RBNode *node, RBRoot *root,
                               const RBAugmentCallbacks *augment)
{
    RBNode *child = node->rb_right;
    RBNode *tmp = node->rb_left;
    RBNode *parent, *rebalance;
    uintptr_t pc;

    if (!tmp) {
        /*
         * Case 1: node to erase has no more than 1 child (easy!)
         *
         * Note that if there is one child it must be red due to 5)
         * and node must be black due to 4). We adjust colors locally
         * so as to bypass rb_erase_color() later on.
         */
        pc = rb_pc(node);
        parent = pc_parent(pc);
        rb_change_child(node, child, parent, root);
        if (child) {
            rb_set_pc(child, pc);
            rebalance = NULL;
        } else {
            rebalance = pc_is_black(pc) ? parent : NULL;
        }
        tmp = parent;
    } else if (!child) {
        /* Still case 1, but this time the child is node->rb_left */
        pc = rb_pc(node);
        parent = pc_parent(pc);
        rb_set_pc(tmp, pc);
        rb_change_child(node, tmp, parent, root);
        rebalance = NULL;
        tmp = parent;
    } else {
        RBNode *successor = child, *child2;
        tmp = child->rb_left;
        if (!tmp) {
            /*
             * Case 2: node's successor is its right child
             *
             *    (n)          (s)
             *    / \          / \
             *  (x) (s)  ->  (x) (c)
             *        \
             *        (c)
             */
            parent = successor;
            child2 = successor->rb_right;

            augment->copy(node, successor);
        } else {
            /*
             * Case 3: node's successor is leftmost under
             * node's right child subtree
             *
             *    (n)          (s)
             *    / \          / \
             *  (x) (y)  ->  (x) (y)
             *      /            /
             *    (p)          (p)
             *    /            /
             *  (s)          (c)
             *    \
             *    (c)
             */
            do {
                parent = successor;
                successor = tmp;
                tmp = tmp->rb_left;
            } while (tmp);
            child2 = successor->rb_right;
            qatomic_set(&parent->rb_left, child2);
            qatomic_set(&successor->rb_right, child);
            rb_set_parent(child, successor);

            augment->copy(node, successor);
            augment->propagate(parent, successor);
        }

        tmp = node->rb_left;
        qatomic_set(&successor->rb_left, tmp);
        rb_set_parent(tmp, successor);

        pc = rb_pc(node);
        tmp = pc_parent(pc);
        rb_change_child(node, successor, tmp, root);

        if (child2) {
            rb_set_parent_color(child2, parent, RB_BLACK);
            rebalance = NULL;
        } else {
            rebalance = rb_is_black(successor) ? parent : NULL;
        }
        rb_set_pc(successor, pc);
        tmp = successor;
    }

    augment->propagate(tmp, NULL);

    if (rebalance) {
        rb_erase_color(rebalance, root, augment);
    }
}

static void rb_erase_augmented_cached(RBNode *node, RBRootLeftCached *root,
                                      const RBAugmentCallbacks *augment)
{
    if (root->rb_leftmost == node) {
        root->rb_leftmost = rb_next(node);
    }
    rb_erase_augmented(node, &root->rb_root, augment);
}


/*
 * Interval trees.
 *
 * Derived from lib/interval_tree.c and its dependencies,
 * especially include/linux/interval_tree_generic.h.
 */

#define rb_to_itree(N)  container_of(N, IntervalTreeNode, rb)

static bool interval_tree_compute_max(IntervalTreeNode *node, bool exit)
{
    IntervalTreeNode *child;
    uint64_t max = node->last;

    if (node->rb.rb_left) {
        child = rb_to_itree(node->rb.rb_left);
        if (child->subtree_last > max) {
            max = child->subtree_last;
        }
    }
    if (node->rb.rb_right) {
        child = rb_to_itree(node->rb.rb_right);
        if (child->subtree_last > max) {
            max = child->subtree_last;
        }
    }
    if (exit && node->subtree_last == max) {
        return true;
    }
    node->subtree_last = max;
    return false;
}

static void interval_tree_propagate(RBNode *rb, RBNode *stop)
{
    while (rb != stop) {
        IntervalTreeNode *node = rb_to_itree(rb);
        if (interval_tree_compute_max(node, true)) {
            break;
        }
        rb = rb_parent(&node->rb);
    }
}

static void interval_tree_copy(RBNode *rb_old, RBNode *rb_new)
{
    IntervalTreeNode *old = rb_to_itree(rb_old);
    IntervalTreeNode *new = rb_to_itree(rb_new);

    new->subtree_last = old->subtree_last;
}

static void interval_tree_rotate(RBNode *rb_old, RBNode *rb_new)
{
    IntervalTreeNode *old = rb_to_itree(rb_old);
    IntervalTreeNode *new = rb_to_itree(rb_new);

    new->subtree_last = old->subtree_last;
    interval_tree_compute_max(old, false);
}

static const RBAugmentCallbacks interval_tree_augment = {
    .propagate = interval_tree_propagate,
    .copy = interval_tree_copy,
    .rotate = interval_tree_rotate,
};

/* Insert / remove interval nodes from the tree */
void interval_tree_insert(IntervalTreeNode *node, IntervalTreeRoot *root)
{
    RBNode **link = &root->rb_root.rb_node, *rb_parent = NULL;
    uint64_t start = node->start, last = node->last;
    IntervalTreeNode *parent;
    bool leftmost = true;

    while (*link) {
        rb_parent = *link;
        parent = rb_to_itree(rb_parent);

        if (parent->subtree_last < last) {
            parent->subtree_last = last;
        }
        if (start < parent->start) {
            link = &parent->rb.rb_left;
        } else {
            link = &parent->rb.rb_right;
            leftmost = false;
        }
    }

    node->subtree_last = last;
    rb_link_node(&node->rb, rb_parent, link);
    rb_insert_augmented_cached(&node->rb, root, leftmost,
                               &interval_tree_augment);
}

void interval_tree_remove(IntervalTreeNode *node, IntervalTreeRoot *root)
{
    rb_erase_augmented_cached(&node->rb, root, &interval_tree_augment);
}

/*
 * Iterate over intervals intersecting [start;last]
 *
 * Note that a node's interval intersects [start;last] iff:
 *   Cond1: node->start <= last
 * and
 *   Cond2: start <= node->last
 */

static IntervalTreeNode *interval_tree_subtree_search(IntervalTreeNode *node,
                                                      uint64_t start,
                                                      uint64_t last)
{
    while (true) {
        /*
         * Loop invariant: start <= node->subtree_last
         * (Cond2 is satisfied by one of the subtree nodes)
         */
        RBNode *tmp = qatomic_read(&node->rb.rb_left);
        if (tmp) {
            IntervalTreeNode *left = rb_to_itree(tmp);

            if (start <= left->subtree_last) {
                /*
                 * Some nodes in left subtree satisfy Cond2.
                 * Iterate to find the leftmost such node N.
                 * If it also satisfies Cond1, that's the
                 * match we are looking for. Otherwise, there
                 * is no matching interval as nodes to the
                 * right of N can't satisfy Cond1 either.
                 */
                node = left;
                continue;
            }
        }
        if (node->start <= last) {         /* Cond1 */
            if (start <= node->last) {     /* Cond2 */
                return node; /* node is leftmost match */
            }
            tmp = qatomic_read(&node->rb.rb_right);
            if (tmp) {
                node = rb_to_itree(tmp);
                if (start <= node->subtree_last) {
                    continue;
                }
            }
        }
        return NULL; /* no match */
    }
}

IntervalTreeNode *interval_tree_iter_first(IntervalTreeRoot *root,
                                           uint64_t start, uint64_t last)
{
    IntervalTreeNode *node, *leftmost;

    if (!root || !root->rb_root.rb_node) {
        return NULL;
    }

    /*
     * Fastpath range intersection/overlap between A: [a0, a1] and
     * B: [b0, b1] is given by:
     *
     *         a0 <= b1 && b0 <= a1
     *
     *  ... where A holds the lock range and B holds the smallest
     * 'start' and largest 'last' in the tree. For the later, we
     * rely on the root node, which by augmented interval tree
     * property, holds the largest value in its last-in-subtree.
     * This allows mitigating some of the tree walk overhead for
     * for non-intersecting ranges, maintained and consulted in O(1).
     */
    node = rb_to_itree(root->rb_root.rb_node);
    if (node->subtree_last < start) {
        return NULL;
    }

    leftmost = rb_to_itree(root->rb_leftmost);
    if (leftmost->start > last) {
        return NULL;
    }

    return interval_tree_subtree_search(node, start, last);
}

IntervalTreeNode *interval_tree_iter_next(IntervalTreeNode *node,
                                          uint64_t start, uint64_t last)
{
    RBNode *rb, *prev;

    rb = qatomic_read(&node->rb.rb_right);
    while (true) {
        /*
         * Loop invariants:
         *   Cond1: node->start <= last
         *   rb == node->rb.rb_right
         *
         * First, search right subtree if suitable
         */
        if (rb) {
            IntervalTreeNode *right = rb_to_itree(rb);

            if (start <= right->subtree_last) {
                return interval_tree_subtree_search(right, start, last);
            }
        }

        /* Move up the tree until we come from a node's left child */
        do {
            rb = rb_parent(&node->rb);
            if (!rb) {
                return NULL;
            }
            prev = &node->rb;
            node = rb_to_itree(rb);
            rb = qatomic_read(&node->rb.rb_right);
        } while (prev == rb);

        /* Check if the node intersects [start;last] */
        if (last < node->start) {  /* !Cond1 */
            return NULL;
        }
        if (start <= node->last) { /* Cond2 */
            return node;
        }
    }
}

/* Occasionally useful for calling from within the debugger. */
#if 0
static void debug_interval_tree_int(IntervalTreeNode *node,
                                    const char *dir, int level)
{
    printf("%4d %*s %s [%" PRIu64 ",%" PRIu64 "] subtree_last:%" PRIu64 "\n",
           level, level + 1, dir, rb_is_red(&node->rb) ? "r" : "b",
           node->start, node->last, node->subtree_last);

    if (node->rb.rb_left) {
        debug_interval_tree_int(rb_to_itree(node->rb.rb_left), "<", level + 1);
    }
    if (node->rb.rb_right) {
        debug_interval_tree_int(rb_to_itree(node->rb.rb_right), ">", level + 1);
    }
}

void debug_interval_tree(IntervalTreeNode *node);
void debug_interval_tree(IntervalTreeNode *node)
{
    if (node) {
        debug_interval_tree_int(node, "*", 0);
    } else {
        printf("null\n");
    }
}
#endif