Sep 29, 2026

[coroutine] event/notify example


#include <coroutine>
#include <deque>
#include <exception>
#include <iostream>
#include <utility>

// Single-threaded scheduler: a queue of coroutines ready to resume.
class Scheduler {
 public:
  void Schedule(std::coroutine_handle<> h) {
    std::cout << "  [queue push_back " << h.address() << "]\n";
    ready_.push_back(h);
  }

  void Run() {
    while (!ready_.empty()) {
      std::coroutine_handle<> h = ready_.front();
      ready_.pop_front();
      std::cout << "  [queue pop_front -> resume " << h.address() << "]\n";
      h.resume();
    }
  }

 private:
  std::deque<std::coroutine_handle<>> ready_;
};

// Coroutine condition variable. Wait() parks the handle;
// Notify*() moves parked handles to the scheduler's ready queue.
class CondVar {
 public:
  explicit CondVar(Scheduler& sched) : sched_(sched) {}

  auto Wait() {
    struct Awaiter {
      CondVar& cv;
      bool await_ready() const noexcept { return false; }
      void await_suspend(std::coroutine_handle<> h) {
        std::cout << "  [cv parks " << h.address() << "]\n";
        cv.waiters_.push_back(h);
      }
      void await_resume() const noexcept {}
    };
    return Awaiter{*this};
  }

  void NotifyOne() {
    if (waiters_.empty()) return;
    sched_.Schedule(waiters_.front());
    waiters_.pop_front();
  }

  void NotifyAll() {
    while (!waiters_.empty()) NotifyOne();
  }

 private:
  Scheduler& sched_;
  std::deque<std::coroutine_handle<>> waiters_;
};

// Fire-and-forget top-level coroutine (unchanged from the original).
struct Task {
  struct promise_type {
    Task get_return_object() { return {}; }
    std::suspend_never initial_suspend() noexcept { return {}; }
    std::suspend_never final_suspend() noexcept { return {}; }
    void return_void() {}
    void unhandled_exception() { std::terminate(); }
  };
};

// Awaitable child coroutine: lazy start + symmetric transfer both ways.
struct Co {
  struct promise_type {
    std::coroutine_handle<> continuation;  // parent to resume when done

    Co get_return_object() {
      return Co{std::coroutine_handle<promise_type>::from_promise(*this)};
    }
    // Don't run until the parent co_awaits us (so continuation is set).
    std::suspend_always initial_suspend() noexcept { return {}; }

    auto final_suspend() noexcept {
      struct Final {
        bool await_ready() noexcept { return false; }
        std::coroutine_handle<> await_suspend(
            std::coroutine_handle<promise_type> h) noexcept {
          return h.promise().continuation;  // jump straight back to parent
        }
        void await_resume() noexcept {}
      };
      return Final{};
    }
    void return_void() {}
    void unhandled_exception() { std::terminate(); }
  };

  explicit Co(std::coroutine_handle<promise_type> h) : h_(h) {}
  Co(Co&& o) noexcept : h_(std::exchange(o.h_, {})) {}
  ~Co() {
    if (h_) h_.destroy();  // frame lives until the co_await expression ends
  }

  bool await_ready() const noexcept { return false; }
  std::coroutine_handle<> await_suspend(std::coroutine_handle<> parent) {
    h_.promise().continuation = parent;
    return h_;  // jump straight into child; no queue
  }
  void await_resume() const noexcept {}

  std::coroutine_handle<promise_type> h_;
};

bool data_ready = false;

Co WaitForData(CondVar& cv, int id) {
  while (!data_ready) {  // Predicate loop, same as std::condition_variable.
    std::cout << "child " << id << " waits\n";
    co_await cv.Wait();
  }
  std::cout << "child " << id << " sees data\n";
}

Task Consumer(CondVar& cv, int id) {
  std::cout << "consumer " << id << " calls child\n";
  co_await WaitForData(cv, id);  // nested co_await
  std::cout << "consumer " << id << " woke up\n";
}

Task Producer(CondVar& cv) {
  data_ready = true;
  std::cout << "producer notifies\n";
  cv.NotifyAll();
  co_return;
}

int main() {
  Scheduler sched;
  CondVar cv(sched);
  Consumer(cv, 1);
  Consumer(cv, 2);
  Producer(cv);
  std::cout << "--- sched.Run() ---\n";
  sched.Run();
}


In a C++20 coroutine, a suspension point is any location where the coroutine can pause execution, save its local state to the heap-allocated coroutine frame, and return control to its caller or resumer.

There are four types of suspension points, split between those you write explicitly in the function body and those the compiler injects automatically around your code.


Where They Live: Compiler Transformation

When write a coroutine, the C++20 compiler rewrites its body into a state machine wrapped in boilerplate that invokes your promise_type. Every coroutine contains at least the two implicit suspension points, plus any explicit ones we write:



// What the compiler generates behind the scenes for your coroutine:
{
    promise_type promise;
    auto return_object = promise.get_return_object();

    // --- 1. IMPLICIT: Initial Suspension Point ---
    co_await promise.initial_suspend();

    try {
        // --- YOUR COROUTINE BODY STARTS HERE ---
        
        // --- 2. EXPLICIT: co_await Suspension Point ---
        auto data = co_await async_read();

        // --- 3. EXPLICIT: co_yield Suspension Point ---
        co_yield data; // Rewritten as: co_await promise.yield_value(data);

        // co_return is NOT a suspension point; it sets the result and jumps to final_suspend
        co_return;     // Calls promise.return_void() and goto final_suspend;
        
        // --- YOUR COROUTINE BODY ENDS HERE ---
    } catch (...) {
        promise.unhandled_exception();
    }

final_suspend:
    // --- 4. IMPLICIT: Final Suspension Point ---
    co_await promise.final_suspend();
} // Coroutine frame is automatically destroyed here ONLY if final_suspend does not suspend

Does a Suspension Point Always Suspend?

Hitting a suspension point means the coroutine may suspend, not that it must suspend. At every co_await (including those generated by co_yield, initial_suspend, and final_suspend), the compiler queries an Awaiter object using a 3-step protocol:

  1. awaiter.await_ready(): Checked first as a fast-path optimization.

    • If it returns true, the result is already available. The coroutine does not suspend and immediately calls await_resume().

    • If it returns false, the coroutine prepares to suspend by saving its instruction pointer and local registers into the coroutine frame.

  2. awaiter.await_suspend(handle): Called right after the coroutine state is saved. Its return type controls what happens next:

    • void: Truly suspends the coroutine and returns control to the caller/resumer.

    • bool: Returning true suspends and returns to the caller; returning false aborts the suspension and immediately resumes the coroutine on the current thread.

    • std::coroutine_handle: Suspends the current coroutine and immediately resumes the returned handle via symmetric transfer (without growing the call stack).

  3. awaiter.await_resume(): Called once the coroutine is resumed (or immediately if await_ready() returned true). Its return value becomes the result of the co_await expression.


#include <coroutine>
#include <deque>
#include <exception>
#include <iostream>
#include <optional>
#include <stop_token>
#include <utility>

// Counts live coroutine frames so main() can prove nothing leaked.
struct FrameCounter {
  static inline int live = 0;
  FrameCounter() { ++live; }
  ~FrameCounter() { --live; }
};<

// Single-threaded scheduler: a queue of coroutines ready to resume.
class Scheduler {
 public:
  void Schedule(std::coroutine_handle<> h) { ready_.push_back(h); }

  // Returns when no coroutine is runnable. After a stop request, that means
  // every task has exited (graceful shutdown), unless one is stuck on
  // something that ignores the stop token.
  void Run() {
    while (!ready_.empty()) {
      std::coroutine_handle<> h = ready_.front();
      ready_.pop_front();
      h.resume();
    }
  }

 private:
  std::deque<std::coroutine_handle<>> ready_;
};

// Re-queues the current coroutine so others get a turn (simulates work).
auto Yield(Scheduler& sched) {
  struct Awaiter {
    Scheduler& sched;
    bool await_ready() const noexcept { return false; }
    void await_suspend(std::coroutine_handle<> h) { sched.Schedule(h); }
    void await_resume() const noexcept {}
  };
  return Awaiter{sched};
}

// Coroutine condition variable with cancellation.
// Wait(token) parks the handle. It is woken by NotifyOne/NotifyAll OR by
// token's stop request, whichever comes first. Callers must re-check their
// predicate and the token after waking (same as std::condition_variable_any).
class CondVar {
 public:
  explicit CondVar(Scheduler& sched) : sched_(sched) {}

  class WaitAwaiter {
   public:
    WaitAwaiter(CondVar& cv, std::stop_token token)
        : cv_(cv), token_(std::move(token)) {}

    // Already stopped: don't suspend at all.
    bool await_ready() const noexcept { return token_.stop_requested(); }

    void await_suspend(std::coroutine_handle<> h) {
      h_ = h;
      cv_.waiters_.push_back(h);
      // Runs OnStop inline if stop was already requested. The callback is
      // deregistered when this awaiter is destroyed (end of co_await).
      on_stop_.emplace(token_, OnStop{this});
    }

    void await_resume() const noexcept {}

   private:
    struct OnStop {
      WaitAwaiter* self;
      void operator()() const noexcept { self->cv_.CancelWait(self->h_); }
    };

    CondVar& cv_;
    std::stop_token token_;
    std::coroutine_handle<> h_;
    std::optional<std::stop_callback<OnStop>> on_stop_;
  };

  WaitAwaiter Wait(std::stop_token token) {
    return WaitAwaiter(*this, std::move(token));
  }

  void NotifyOne() {
    if (waiters_.empty()) return;
    sched_.Schedule(waiters_.front());
    waiters_.pop_front();
  }

  void NotifyAll() {
    while (!waiters_.empty()) NotifyOne();
  }

 private:
  // Wakes h only if it is still parked here. If NotifyOne already moved it
  // to the ready queue, do nothing; scheduling it twice would resume a
  // coroutine that is not suspended (UB).
  void CancelWait(std::coroutine_handle<> h) {
    for (auto it = waiters_.begin(); it != waiters_.end(); ++it) {
      if (*it == h) {
        waiters_.erase(it);
        sched_.Schedule(h);
        return;
      }
    }
  }

  Scheduler& sched_;
  std::deque<std::coroutine_handle<>> waiters_;
};

// Fire-and-forget top-level coroutine. Frame self-destroys at the end.
struct Task {
  struct promise_type : FrameCounter {
    Task get_return_object() { return {}; }
    std::suspend_never initial_suspend() noexcept { return {}; }
    std::suspend_never final_suspend() noexcept { return {}; }
    void return_void() {}
    void unhandled_exception() { std::terminate(); }
  };
};

// Awaitable child coroutine returning T: lazy start + symmetric transfer.
template <typename T>
class Co {
 public:
  struct promise_type : FrameCounter {
    std::coroutine_handle<> continuation;  // parent to resume when done
    std::optional<T> result;

    Co get_return_object() {
      return Co(std::coroutine_handle<promise_type>::from_promise(*this));
    }
    std::suspend_always initial_suspend() noexcept { return {}; }
    auto final_suspend() noexcept {
      struct Final {
        bool await_ready() noexcept { return false; }
        std::coroutine_handle<> await_suspend(
            std::coroutine_handle<promise_type> h) noexcept {
          return h.promise().continuation;  // jump straight back to parent
        }
        void await_resume() noexcept {}
      };
      return Final{};
    }
    void return_value(T v) { result.emplace(std::move(v)); }
    void unhandled_exception() { std::terminate(); }
  };

  explicit Co(std::coroutine_handle<promise_type> h) : h_(h) {}
  Co(Co&& o) noexcept : h_(std::exchange(o.h_, {})) {}
  ~Co() {
    if (h_) h_.destroy();
  }

  bool await_ready() const noexcept { return false; }
  std::coroutine_handle<> await_suspend(std::coroutine_handle<> parent) {
    h_.promise().continuation = parent;
    return h_;  // jump straight into child; no queue
  }
  T await_resume() { return std::move(*h_.promise().result); }

 private:
  std::coroutine_handle<promise_type> h_;
};

// Unbounded work queue.
class Channel {
 public:
  explicit Channel(Scheduler& sched) : cv_(sched) {}

  void Push(int v) {
    items_.push_back(v);
    cv_.NotifyOne();
  }

  // Graceful semantics: keeps handing out queued items even after stop, and
  // returns nullopt only when the queue is empty AND stop was requested.
  Co<std::optional<int>> Pop(std::stop_token token) {
    while (items_.empty()) {
      if (token.stop_requested()) co_return std::nullopt;
      co_await cv_.Wait(token);  // nested co_await; parks THIS child frame
    }
    int v = items_.front();
    items_.pop_front();
    co_return v;
  }

 private:
  std::deque<int> items_;
  CondVar cv_;
};

Task Consumer(Scheduler& sched, Channel& ch, int id, std::stop_token token) {
  struct Cleanup {  // proves destructors run on shutdown
    int id;
    ~Cleanup() { std::cout << "consumer " << id << " cleanup (RAII)\n"; }
  } cleanup{id};

  while (std::optional<int> item = co_await ch.Pop(token)) {
    std::cout << "consumer " << id << " processes " << *item << "\n";
    co_await Yield(sched);  // simulate work
  }
  std::cout << "consumer " << id << " exits\n";
}

Task Producer(Scheduler& sched, Channel& ch, std::stop_token token) {
  for (int i = 0; !token.stop_requested(); ++i) {
    std::cout << "producer pushes " << i << "\n";
    ch.Push(i);
    co_await Yield(sched);
  }
  std::cout << "producer exits (stop requested)\n";
}

// Stands in for a SIGTERM handler / deadline: requests stop after `ticks`.
Task StopAfter(Scheduler& sched, std::stop_source& stop, int ticks) {
  for (int i = 0; i < ticks; ++i) co_await Yield(sched);
  std::cout << "=== shutdown requested ===\n";
  stop.request_stop();  // wakes parked waiters via their stop_callbacks
}

int main() {
  Scheduler sched;
  Channel ch(sched);
  std::stop_source stop;

  Consumer(sched, ch, 1, stop.get_token());
  Consumer(sched, ch, 2, stop.get_token());
  Producer(sched, ch, stop.get_token());
  StopAfter(sched, stop, 3);

  sched.Run();  // returns once every task has exited

  std::cout << "live coroutine frames after Run(): " << FrameCounter::live
            << "\n";
  return FrameCounter::live == 0 ? 0 : 1;
}

Sep 28, 2026

Where Did My Stack Frame Go? Blocking Tail Calls in Clang and GCC

A crash in g shows a backtrace that jumps straight from main to g, even though main called f and f called g. The debugger is fine. The compiler removed f on purpose.

Tail-call optimization

int g(void);
int f(void) { return g(); }
Calling g is the last thing f does, so f no longer needs its stack frame. At -O2, Clang and GCC compile f to a single jump:
f:
    jmp g
This saves a stack frame and a return. It also removes f from the stack, which breaks: 
  • Backtraces: crash reports and debuggers leave out f. 
  • Profilers: g's time is charged to f's caller. Stack walkers: a logger that skips N frames reports the wrong caller.

Three ways to keep the frame

1. Empty asm volatile after the call (one call site, Clang and GCC)
int f(void) {
  int r = g();
  __asm__ __volatile__("");
  return r;
}
The empty asm produces no instructions. It is volatile, so the compiler can neither delete it nor move it before the call. Because it runs after g() returns, g() is no longer in tail position:
f:
    subq $8, %rsp
    call g
    addq $8, %rsp
    ret
2. __attribute__((disable_tail_calls)) (one function, Clang only)
__attribute__((disable_tail_calls))
int f(void) { return g(); }
This states the intent clearly and covers every call inside f.

3. -fno-optimize-sibling-calls (whole file, Clang and GCC) This flag fits profiling builds. For a single function, it turns off too much. Inlining still removes frames
If the compiler inlines f into its caller, f has no frame to keep. When the frame must exist, also add __attribute__((noinline)).
 

Cost

Each blocked tail call costs a call/ret pair and one stack frame instead of a jmp. This matters only on very hot paths, or in deep recursion that relied on tail calls to avoid overflowing the stack.

Demo

Paste this into Compiler Explorer (godbolt.org) and compile with -O2:
int g(void);
int with_tail_call(void)    { return g(); }
int without_tail_call(void) { int r = g(); __asm__ __volatile__(""); return r; }
with_tail_call compiles to jmp g. without_tail_call compiles to call g followed by ret. 
Verified with Clang 21 and GCC 15 on x86-64.

Takeaway

  • One call site: add __asm__ __volatile__("") after the call.
  • One function (Clang): use __attribute__((disable_tail_calls)).
  • A whole file: compile with -fno-optimize-sibling-calls.
  • To guarantee the frame: also add noinline.

Sep 27, 2026

[this is the way]

 The impediment to action advances action. What stands in the way becomes the way. - Marcus Aurelius

Sep 12, 2026

C++ template code COMDAT folding

Reference:
Reducing C++ template bloat by factoring out the type-dependent portions of the function


COMDAT folding (also known as Identical Code Folding or ICF/Safe ICF) is a link-time optimization where the linker detects functions or read-only data sections that compile into byte-for-byte identical machine code, merges them into a single instance, and points all call sites to that unified copy.

When instantiates templates over types that share the same underlying memory layout and operations, the compiler often emits redundant assembly:

template <typename T>
void push_item(T* item) {
    // Operations on item...
}

// In translation unit A:
push_item<Dog>(dog_ptr);

// In translation unit B:
push_item<Cat>(cat_ptr);
Because Dog* and Cat* are both raw machine pointers (typically 8 bytes), the generated machine instructions for push_item and push_item are identical. 

Normally, C++ templates emit code into COMDAT sections (Common Data sections) with "link-once" semantics (select any). This allows the linker to discard duplicates of push_item compiled across multiple .cpp files. However, normal deduplication only matches identical symbol names. It cannot eliminate the identical assembly between push_item and push_item because their mangled symbol names differ.

How COMDAT Folding Works

During the link step, the linker inspects all candidate sections:

Content Comparison: The linker compares the byte streams, relocation targets, and alignment requirements of functions marked as foldable (typically COMDAT sections generated by inline functions, template instantiations, and virtual tables).

Section Merging: If two distinct functions (e.g., std::vector<int*>::size() and std::vector<char*>::size()) produce identical instructions and reference identical offsets, the linker discards one function body.

Symbol Redirection: The symbol table entry for the discarded function is updated to point directly to the entry point of the retained function.

Key Benefits

Reduced Binary Footprint: Prevents template-heavy code (like std::vector<T*> for hundreds of pointer types) from inflating executable size.

Instruction Cache (I-Cache) Efficiency: Multiple logically distinct types share hot cache lines instead of thrashing the instruction cache with duplicate code.


The Function Pointer Quirk

Under strict C++ rules, every distinct function must have a unique address:

assert(&push_item<Dog> != &push_item<Cat>);

When COMDAT folding collapses these functions, &push_item<Dog> == &push_item<Cat> evaluates to true. This can break code that relies on function pointer uniqueness for type-tagging or callback registries.

To handle this, linkers provide different safety levels:
LLVM (lld)  --icf=safe Only merges functions whose addresses are never taken, preserving C++ address-uniqueness guarantees.
LLVM (lld)  --icf=all Aggressively merges all identical functions, even if addresses are taken.


Instead of generating redundant machine code for hundreds of pointer instantiations and hoping the linker cleans it up with COMDAT folding / ICF, standard library implementations and runtime systems use type erasure with non-templated (or void-pointer-based) base classes.

By shifting the heavy procedural logic into a shared base class, the exposed template becomes a thin, inline wrapper that carries zero runtime binary overhead.

The Architecture: Base-Derived Split

The pattern divides a container or utility into two layers:

The Erased Base Class: Implements memory allocation, capacity resizing, buffer shifts, and element index arithmetic using void* or raw byte buffers. This code is compiled once into the runtime library or translation unit.

The Typed Template Wrapper: Derives from or wraps the base class. It only exposes strongly typed interfaces, using zero-cost casts (reinterpret_cast or static_cast) to translate between T* and void*.


// --- Shared Implementation (compiled once into binary/lib) ---
class VectorPtrBase {
protected:
    void** data_ = nullptr;
    size_t size_ = 0;
    size_t capacity_ = 0;

    void grow_and_insert(size_t index, void* element) {
        // All heavy buffer reallocation, index shifting,
        // and boundary checks happen HERE once.
        if (size_ == capacity_) {
            size_t new_cap = capacity_ == 0 ? 8 : capacity_ * 2;
            void** new_data = new void*[new_cap];
            for (size_t i = 0; i < size_; ++i) new_data[i] = data_[i];
            delete[] data_;
            data_ = new_data;
            capacity_ = new_cap;
        }
        data_[index] = element;
        ++size_;
    }

    void* get_element(size_t index) const {
        return data_[index];
    }
};

// --- Thin Typed Wrapper (specialization for any pointer type) ---
template <typename T>
class Vector<T*> : private VectorPtrBase {
public:
    void push_back(T* val) {
        // Zero-cost static cast; inlines down to a direct call to the base
        grow_and_insert(size_, static_cast<void*>(val));
    }

    T* operator[](size_t index) const {
        return static_cast<T*>(get_element(index));
    }

    size_t size() const { return size_; }
};

Why This Beats Relying Purely on COMDAT Folding

Faster Compilation Times: The compiler does not have to parse, instantiate, type-check, and generate intermediate representation (IR) / assembly for grow_and_insert across Vector<Apple*>, Vector<Banana*>, and Vector<Car*>.

Lower Linker Overhead: Linker-time ICF requires analyzing section hashes, inspecting instruction bytes, and walking relocation tables to prove equivalence. Pointer erasure eliminates the duplicate sections before the object files reach the linker.

Guaranteed Code Sharing: COMDAT folding is sensitive to subtle differences (such as debug information emission, compiler optimization levels, or platform-specific pointer calling conventions). The base-class approach enforces code sharing by construction.

Preserved Pointer Address Guarantees: Because the wrapper's member functions inline entirely into the caller or delegate to the shared base, it avoids issues where --icf=safe refuses to fold functions whose addresses were taken.

Sep 11, 2026

[left-right datastructure] The Cost of Concurrency Coordination

I used to use this dual-copy/pointer swap trick for Linkedin's ATS server.

Reference:
The Cost of Concurrency Coordination with Jon Gjengset
[low latency] What is low latency (definition)

In concurrent systems programming, conventional wisdom often reduces synchronization performance to simple rules of thumb:

  • "Mutexes are slow because threads get descheduled."
  • "If your workload is read-heavy, swap out your Mutex for a Reader-Writer Lock (RwLock)."
  • "Lock-free data structures are inherently fast."
Experience engineer should know these statements aren't true.

It really based on the workload patterns. (DoD)

The RwLock Trap

When developers see read contention on a mutex, the standard instinct is reaching for a Reader-Writer Lock (std::sync::RwLock in Rust, std::shared_mutex in C++). In theory, multiple readers are non-exclusive and should execute cleanly in parallel.

In practice, the benchmark reveals something surprising:
The RwLock read throughput starts roughly equal to the Mutex.
As the number of concurrent reader threads grows, RwLock performance degrades faster and ends up significantly worse than the basic Mutex.


The Hardware Reality: Cache Lines and the MESI Protocol


Why RwLock::read() is Actually a Write

Here lies the catch: Taking a read lock is not a read operation.

To track how many active readers hold the lock, an RwLock maintains an internal reader counter. When a reader calls read(), the CPU must execute an atomic increment (fetch_add) on that shared counter.

Core 0 (Reader 1)   --> fetch_add(reader_count) --> Needs Exclusive Ownership (Line in 'M')
Core 1 (Reader 2)   --> fetch_add(reader_count) --> Forces Core 0 invalidation; pulls line to Core 1
Core 0 (Releasing)  --> fetch_sub(reader_count) --> Pulls line back from Core 1

Every single acquire and release triggers cache-line bouncing across the processor interconnect. Each transition costs ~30 ns. Across acquire and release, a thread spends ~60 ns just synchronizing the lock state—over half the latency of a trip out to main RAM.




Why does Mutex hold up better under extreme thread contention than RwLock?

With a Mutex, only the current owner touches the lock line. Other threads block or queue up in sequence.
With an RwLock, dozens of reader threads aggressively hammer the exact same counter simultaneously, creating intense cache-line ping-pong across every core.


The Left-Right Data Structure: Coordination Without Contention

To make readers truly scalable, readers must never write to a shared cache line.

Jon presents Left-Right, a concurrency primitive, designed for workloads with frequent reads and infrequent writes (e.g., in-memory key-value lookups, routing tables, configuration maps).

High-Level Architecture

Instead of locking access to a single instance, Left-Right maintains two identical copies of the underlying data structure (the Left copy and the Right copy), mediated by an atomic pointer.



The Read Path (Wait-Free & Shared-State Free)

Every reader thread is registered with a private, thread-local counter aligned to its own cache line.
When reading:
  • The reader announces its entry by updating its own thread-local counter.
  • It loads the global atomic pointer to find the current active copy (Left or Right).
  • It executes the read directly on that copy without taking any locks.
  • It signals completion on its private counter.
  • Because each reader modifies only its own cache line, there is zero cross-core invalidation between readers. Their cache lines stay in the Exclusive/Modified state within their local L1/L2 caches.

The Write Path (Two-Phase Reconciliation)

  • Step 1: Writer mutates Right Copy (inactive)
  • Step 2: Writer swaps Atomic Pointer to Right
  • Step 3: Writer waits for readers in Left Copy to exit (epochs/counters)
  • Step 4: Writer replays mutations onto Left Copy
Mutate Inactive Copy: The writer applies the update to the copy that readers aren't currently directed to (e.g., the Right copy).
Atomic Pointer Swap: The writer atomically swings the pointer to Right. All new incoming read operations will now read from Right.
Wait for Old Readers: Readers that entered before the pointer swap are still safely executing inside the Left copy. The writer scans the per-thread counters in a loop until it confirms that all readers active during the switch have finished.
Replay & Synchronize: Once the Left copy is completely drained of readers, the writer replays the exact same update from an operational log onto the Left copy. Both copies are now identical again.

The "Four-Core Drop": A Real-World Lesson in False Sharing

While benchmarking Left-Right, Jon observed expected linear scalability up to three cores—then suddenly, at four cores, throughput plummeted by nearly an order of magnitude:

Throughput
    ^
    |         /
    |       /
    |     /   <-- Expected linear scaling
    |   /
    |  *
    |       |     \__ * <-- Plummeted 10x at 4 cores!
    +----------------------------------------> Cores

The bug wasn't an algorithmic flaw or a NUMA boundary traversal. It was False Sharing:
  • The internal implementation stored per-thread reader counters together in an array.
  • Multiple 64-bit counter values fit inside a single 64-byte cache line (8 bytes × 8 = 64 bytes).
  • Even though Core 0 and Core 1 were updating completely independent counter variables, those variables shared the exact same physical cache line. The CPU was forced to ping-pong the line between cores on every single reader check-in.

The Fix

Enforce cache-line alignment on the counter type:

#[repr(align(64))]
struct AlignedReaderCounter {
    counter: AtomicUsize,
}

By ensuring each thread's counter lived on its own dedicated 64-byte boundary, the false sharing vanished, and performance restored to ~3 billion reads/second across 10 cores—scaling linearly.

Engineering Trade-offs: When Should You Use Left-Right?

Left-Right is not a magic drop-in replacement for every concurrency scenario. It trades memory and write performance for extreme read throughput:

ConstraintLeft-Right Trade-Off
Memory FootprintDoubled (2×), because two full copies of the data structure must live in memory.
Write OverheadHigh. Writers must apply changes twice (once per copy), keep an operation log, and wait for reader epochs to drain.
Write ConcurrencySingle writer only. Multiple concurrent writers require an external lock.
Consistency ModelEventually consistent / Non-linearizable. Readers might see slightly stale data before a pointer swap, and writers cannot immediately read-your-own-writes from the reader handle.
DeterminismOperations must be completely deterministic so that replaying them on the second copy yields identical internal state.


Summary Takeaways

  • "Lock-Free" does not mean "Contention-Free": Eliminating OS-level locks doesn't matter if your threads are repeatedly modifying the same atomic variable on a single shared cache line.
  • Short critical sections expose synchronization overhead: If your protected work takes 5 ns (like a hash map lookup) and your lock acquire/release costs 60 ns in cache line bounces, synchronization dominates your runtime.
  • Align for the hardware: Always guard against false sharing in concurrent per-thread arrays using 64-byte alignment (alignas(64) in C++, #[repr(align(64))] in Rust).
  • Tailor algorithms to your access patterns: When your system is 99% reads and you can afford the memory overhead and deterministic write logs, patterns like Left-Right turn cache lines from a bottleneck into an advantage.

Aug 4, 2026

[CppNow][summary] Lock-free Programming is Dead - Long Live Lock-free Programming! - Fedor G Pikus - C++Now 2026

Deep Dive Summary: Lock-free Programming is Dead - Long Live Lock-free Programming!

Speaker: Fedor Pikus
Conference: C++Now 2026
Topic: Microarchitecture, Concurrency, Lock-Free Data Structures, and Low-Level C++ Optimization

Reference: 


Executive Overview

For decades, modern C++ concurrent programming followed a simple, widely accepted rule: Lock-free algorithms should be used for high-contention, performance-critical paths, while locks should be reserved for low-contention or non-critical code.

In this landmark C++Now 2026 presentation, Fedor Pikus demonstrates that advancements in modern CPU microarchitecture (x86-64 Intel/AMD, ARM Grace/Graviton, Apple Silicon) have completely inverted this paradigm:

  1. At High Contention: Properly engineered spin locks outperform lock-free (compare_exchange loops) and wait-free (fetch_add) atomic algorithms.
  2. At Low Contention: Taking a spin lock—even once every 100 iterations—severely degrades surrounding CPU out-of-order execution performance ("pipeline poisoning"), whereas atomics excel without stalling execution pipelines.
  3. The Optimal Modern Paradigm: High-performance concurrent data structures must combine both techniques—utilizing custom spin locks for high-contention index allocation/state transitions and atomic handoffs for low-contention payload access.

Progress Guarantees & Theoretical Definitions

Before diving into hardware microarchitecture, Pikus clarifies the classical computer science definitions of thread progress:

Guarantee CS Definition Typical C++ Primitive Hardware Reality
Wait-Free Every thread completes its operation in a bounded number of algorithmic steps. std::atomic::fetch_add Not constant time. Hardware cache-line invalidation forces sequential memory execution.
Lock-Free At least one thread makes progress overall; losers retry in a loop. std::atomic::compare_exchange_weak / strong High contention causes severe CAS-retry loops and cache coherence thrashing.
Lock-Based One thread holds exclusive access; all other contending threads wait/block. std::mutex, Custom Spin Locks OS mutexes incur context switch overhead, but well-tuned spin locks eliminate CAS retry overhead.

Key Distinction: Computer science definitions measure algorithmic steps, not CPU clock cycles. A "wait-free" instruction executes a single instruction step, but at the hardware level, cache coherency mechanisms force memory accesses to serialize, causing hardware-level waiting.


Microarchitectural Deep Dive: Why Atomics Fail Under High Contention

1. Cache Coherency and Read-For-Ownership (RFO)

Modifying any atomic variable requires exclusive access to its underlying 64-byte cache line:

  • To write to a cache line, a CPU core must issue a Read-For-Ownership (RFO) request across the interconnect.
  • The requesting core must wait for all other cores holding that cache line in a Shared (S) state to invalidate their local L1/L2 caches and send back an Acknowledgment (ACK) signal.
    (MESI)
  • Under heavy multi-threaded contention, cores spend the majority of their clock cycles waiting for speed-of-light electrical signal propagation across the chip to complete RFO invalidation handshakes.

2. True Sharing vs. False Sharing

Experimentation proves that false sharing (multiple independent atomics residing on the same 64-byte cache line) and true sharing (all threads hammering the exact same atomic variable) suffer from the exact same latency penalty at high thread counts. The hardware bottleneck is the cache line granularity, not the specific integer being modified.


Engineering a High-Performance Spin Lock

To outperform atomics at high contention, a spin lock must be engineered specifically to respect hardware cache coherency protocols.

Critical Design Features:

  1. Pre-Read Probe (Test-and-Test-and-Set):
    • Before attempting an expensive atomic operation (atomic_exchange), the thread performs a relaxed, read-only load of the lock flag.
    • Reading allows the core to acquire the cache line in a Shared (S) state without revoking exclusive access from the thread currently holding the lock.
  2. Pre-Read Retries:
    • Performing ~8 relaxed reads on x86 before attempting an atomic swap ensures that in-flight RFO signals have time to settle, preventing premature cache-line stealing.
  3. Aggressive Back-off:
    • Unlocking a spin lock requires writing 0 to memory, which itself requires an RFO request. If waiting threads continuously hammer the lock with atomic writes, they steal the cache line from the lock holder, severely delaying the unlock operation.
    • Back-off logic (or yielding) keeps waiting threads from stealing the cache line during lock release.

The Low-Contention Asymmetry: "Pipeline Poisoning"

When benchmarking code that mixes parallel payload computation (e.g., local mathematical tasks) with synchronization (e.g., updating a shared counter):

  • High Contention Domain: Spin locks deliver up to 2.5x higher overall application throughput compared to atomics.
  • Low Contention Domain: When synchronization occurs rarely (e.g., 1 out of 100 iterations), using a spin lock causes program throughput to plummet compared to atomics.
High Contention (1:1 Parallel to Shared Work):
[Spin Lock]  ========================> 2.5x Throughput vs Atomics
[Atomic CAS] ========> 1.0x

Low Contention (100:1 Parallel to Shared Work):
[Atomic CAS] ========================> 1.0x (Fast Execution)
[Spin Lock]  =====> 0.25x (Severe Pipeline Poisoning)

Hardware Profiling & Root Cause Analysis (Intel VTune / Linux Perf)

By inspecting CPU hardware performance counters (RESOURCE_STALLS.STORE_BUFFER), Pikus identified the exact microarchitectural bottleneck:

  1. Store Buffer Stalls: Spin locks force massive store buffer stalls in x86 execution pipelines. On x86 architectures, memory writes exit the store buffer in strict program retirement order.
  2. Implicit vs. Explicit Dependencies:
    • Atomics (fetch_add, CAS): Fuse control and data into a single variable. The CPU out-of-order execution engine can inspect the instruction stream and recognize explicit data dependencies, allowing non-dependent payload instructions to flow around the atomic operation.
    • Locks: Separate control (the lock flag) from data (the guarded payload). Because the CPU cannot reason through implicit memory dependencies across bidirectional acquire/release memory barriers, it cannot predict safety across the critical section. As a result, the out-of-order execution pipeline flushes and stalls until the lock and unlock operations fully retire.

Practical Application: The Hybrid MPMC Ring-Buffer Queue

To prove these findings, Fedor Pikus constructed a high-throughput Multi-Producer Multi-Consumer (MPMC) ring-buffer queue designed around hybrid synchronization:

+-----------------------------------------------------------------------+
|                         MPMC QUEUE DESIGN                             |
+-----------------------------------------------------------------------+
| High-Contention Domain  -->  Spin Lock (Guards Head / Tail Indices)  |
| Separate Cache Lines    -->  Prevents Producer/Consumer Thrashing     |
+-----------------------------------------------------------------------+
| Low-Contention Domain   -->  Atomic Slot Keys (std::atomic<Key>)    |
| Exclusive Slot Access   -->  Zero Lock Overhead for Payload Handoff   |
+-----------------------------------------------------------------------+

Performance Benchmark Results:

  • Throughput: Dramatically outperforms traditional pure lock-free MPMC queues across Intel Granite Rapids, AMD Zen 5, Nvidia Grace, and Apple Silicon.
  • Average Latency: Significantly lower mean and 95th/99th percentile latency compared to pure lock-free implementations.
  • Tail Latency Exception (99.99%+): Pure lock-free queues only win at extreme tail latencies where thread preemption risks affect lock-based structures.

Duration & Power Considerations: When Spin Locks Fail

While spin locks excel at short critical sections, they carry severe operational trade-offs if critical section processing times grow long:

  1. Busy-Waiting Energy Waste: A thread waiting on a spin lock burns 100% CPU core utilization, drawing maximum power, generating heat, and causing thermal throttling.
  2. Thread Preemption Disaster: If the thread holding a spin lock is preempted by the OS scheduler (context-switched out), all contending threads will actively spin for their entire OS time slice doing zero productive work while waiting for the lock holder to be rescheduled.
  3. The Industrial Solution (Adaptive / Two-Phase Locks): Production systems (e.g., database kernels, runtime engines) utilize adaptive locks:
    • Phase 1: Spin for a short, bounded duration (~50–100 iterations using CPU pause hints like _mm_pause() or YIELD).
    • Phase 2: If the lock remains unacquired, yield execution to the OS kernel via a futex sleep system call.

Architectural Anomalies & Micro-architectural Surprises

  1. Cache-Line Separation of Lock and Payload: Placing the spin lock flag and the guarded data variable on different cache lines improves high-contention throughput. It prevents waiting threads (doing relaxed pre-reads on the lock flag) from invalidating the lock holder's cache line while it modifies the payload variable.
  2. AMD Zen 4 / Zen 5 Near-Memory Atomics: AMD Zen 4 introduced hardware execution ALUs directly inside the L3 memory controller. If a core attempts a fetch_add without owning the cache line, it offloads the operation directly to the L3 controller, bypassing L1/L2 cache-line invalidation cycles.
  3. Apple Silicon CAS Back-off: Apple M-series chips use a power-efficient, high-latency directory interconnect. Implementing explicit back-off inside a compare_exchange loop on Apple Silicon improves throughput by 10x, elevating CAS performance close to spin lock levels.

Summary Principles for Modern C++ Concurrency

  1. High Contention: Abandon pure lock-free CAS loops. Use properly engineered spin locks featuring relaxed pre-read probes, iteration limits, and back-off logic.
  2. Low Contention: Avoid locks completely. Use atomic primitives (std::atomic) to prevent CPU store buffer flushes and out-of-order pipeline stalls.
  3. Hybrid Architecture: Structure concurrent data structures to use spin locks for high-contention indexing (e.g., ring-buffer head/tail allocation) and atomic state flags for low-contention data transfer.
  4. Bounded Work: Keep spin lock critical sections strictly bounded to a few dozen nanoseconds; fall back to adaptive OS-backed locks (futex) if processing times can exceed thread time-slices.

#include <atomic>
#include <chrono>
#include <new>
#include <thread>

#if defined(__x86_64__) || defined(_M_X64)
#include <emmintrin.h> // For _mm_pause()
#endif

// Align to prevent false sharing with adjacent cache lines
class alignas(std::hardware_destructive_interference_size) AdaptiveSpinLock {
public:
    AdaptiveSpinLock() noexcept = default;

    // Non-copyable, non-movable
    AdaptiveSpinLock(const AdaptiveSpinLock&) = delete;
    AdaptiveSpinLock& operator=(const AdaptiveSpinLock&) = delete;

    void lock() noexcept {
        // Phase 1: Fast path (Uncontended)
        if (!state_.exchange(true, std::memory_order_acquire)) {
            return;
        }

        // Phase 2: Active Spin Loop with Pre-Read (Test-and-Test-and-Set)
        int spin_count = 0;
        constexpr int MAX_SPINS = 64; // Max active CPU spinning iterations

        while (true) {
            // Pre-read probe: Read in a relaxed loop to stay in 'Shared' (S) cache state.
            // Avoids sending RFO (Read-For-Ownership) signals while waiting.
            while (state_.load(std::memory_order_relaxed)) {
                pause_cpu(spin_count);

                // Phase 3: Adaptive Fallback to OS-backed Sleeping
                // If spinning takes too long, stop burning CPU/power and park the thread.
                if (++spin_count > MAX_SPINS) {
                    // C++20/23 std::atomic::wait uses OS futex/synch primitives natively
                    state_.wait(true, std::memory_order_relaxed);
                }
            }

            // Attempt to acquire lock via atomic exchange once pre-read detects key is free
            if (!state_.exchange(true, std::memory_order_acquire)) {
                return; // Lock acquired successfully
            }
        }
    }

    bool try_lock() noexcept {
        // First check relaxed state to avoid cache invalidation on failure
        if (state_.load(std::memory_order_relaxed)) {
            return false;
        }
        return !state_.exchange(true, std::memory_order_acquire);
    }

    void unlock() noexcept {
        // Unconditionally release lock
        state_.store(false, std::memory_order_release);

        // Notify any waiting thread that was parked in state_.wait()
        state_.notify_one();
    }

private:
    // Flushes execution pipeline & saves power during short spinning
    static void pause_cpu(int spin_count) noexcept {
        if (spin_count < 16) {
#if defined(__x86_64__) || defined(_M_X64)
            _mm_pause(); // Informs x86 pipeline of spin loop; reduces power & store buffer pressure
#elif defined(__aarch64__)
            asm volatile("yield" ::: "memory"); // ARM yield hint
#endif
        } else {
            // Hint to OS scheduler to run other threads on the same core/time-slice
            std::this_thread::yield();
        }
    }

    // False = Unlocked, True = Locked
    std::atomic<bool> state_{false};
};