Skip to content

Allow isolated non-sample nodes in all dating methods - #531

Open
hyanwong wants to merge 1 commit into
tskit-dev:mainfrom
hyanwong:isolated-nodes
Open

hyanwong wants to merge 1 commit into
tskit-dev:mainfrom
hyanwong:isolated-nodes

Conversation

@hyanwong

@hyanwong hyanwong commented Nov 4, 2026

Copy link
Copy Markdown
Member

Non-sample nodes that are in no edge can't be dated, but previously caused an error (variational_gamma: "disconnected nodes"; inside_outside/maximization: a misleading "not simplified" error).

Now, for all methods:

  • Isolated non-sample nodes keep their input time.
  • Their mn/vr node metadata is NaN, as are their rows in variational_gamma's fit.node_posteriors() (inside_outside already gives NaN for fixed nodes).
  • They no longer influence the dating of other nodes. This needed a fix in rescaling.mutational_area, which built time epochs from all nodes, so an isolated node with a large time distorted the rescaling.
  • Isolated samples still raise an error.

Tests: TestIsolatedNodes in tests/test_inference.py (all three methods, several isolated times; checks other node times are unchanged vs. the tree sequence without the node). Docs and changelog updated. Full suite passes; timing on 500 samples x 10 Mb is unchanged.

Non-sample nodes that are in no edge can't be dated. They now keep their
input time, have NaN posterior mean/variance in node metadata and in
variational_gamma's fit.node_posteriors(), and no longer affect the dating
of other nodes (mutational_area ignores edgeless nodes when building
epochs). Isolated samples still raise an error.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
@nspope

nspope commented Nov 7, 2026

Copy link
Copy Markdown
Contributor

Sorry for delay here, I'll get to a review after the 13th

@hyanwong

hyanwong commented Nov 7, 2026

Copy link
Copy Markdown
Member Author

Thanks. I don't think there's anything controversial here, but it would be good to have an extra set of eyes on it.

@nspope

nspope commented Nov 18, 2026

Copy link
Copy Markdown
Contributor

We need some equivalence testing here, I think ("are estimated node times the same for the same ARG with and without isolated nodes"). Something like:

import msprime
import numpy as np
import pytest
import tskit

import tsdate


def insert_isolated_nodes(ts, times, before):
    """
    Return (tables, new_ids, old_to_new) where isolated non-sample nodes with the
    given times are inserted into the node table immediately before the (original)
    node ids in `before`.
    """
    tables = ts.dump_tables()
    old_nodes = tables.nodes.copy()
    tables.nodes.clear()
    insert = {}
    for b, t in zip(before, times):
        insert.setdefault(b, []).append(t)
    old_to_new = np.full(ts.num_nodes, tskit.NULL, dtype=np.int32)
    new_ids = []
    for i in range(ts.num_nodes + 1):
        for t in insert.get(i, []):
            new_ids.append(tables.nodes.add_row(flags=0, time=t))
        if i < ts.num_nodes:
            old_to_new[i] = tables.nodes.append(old_nodes[i])
    tables.edges.parent = old_to_new[tables.edges.parent]
    tables.edges.child = old_to_new[tables.edges.child]
    tables.mutations.node = old_to_new[tables.mutations.node]
    return tables, np.array(new_ids), old_to_new


METHODS = {
    "variational_gamma": lambda ts, **kw: tsdate.variational_gamma(
        ts, mutation_rate=1e-8, **kw
    ),
    "inside_outside": lambda ts, **kw: tsdate.inside_outside(
        ts, mutation_rate=1e-8, population_size=1e4, **kw
    ),
    "maximization": lambda ts, **kw: tsdate.maximization(
        ts, mutation_rate=1e-8, population_size=1e4, **kw
    ),
}

# (times, position in node table as fraction of num_nodes; 1.0 == append)
PLACEMENTS = {
    "append": ([5.5], [1.0]),
    "prepend": ([5.5], [0.0]),
    "middle_multiple": ([0.0, 7.0, 1e7], [0.5, 0.5, 0.25]),
    "above_root": ([1e9], [0.3]),
    "many": (list(np.linspace(0, 5e4, 50)), list(np.linspace(0, 1, 50))),
}


class TestIsolatedEquivalence:
    @staticmethod
    def simulate(num_samples, seed):
        ts = msprime.sim_ancestry(
            num_samples,
            sequence_length=2e5,
            recombination_rate=1e-8,
            population_size=1e4,
            random_seed=seed,
        )
        return msprime.sim_mutations(ts, rate=1e-8, random_seed=seed)

    @pytest.mark.parametrize("method", METHODS)
    @pytest.mark.parametrize("placement", PLACEMENTS)
    @pytest.mark.parametrize("extra", [{}, {"rescaling_intervals": 5}])
    def test_dating_is_equivalent(self, method, placement, extra):
        if extra and method != "variational_gamma":
            pytest.skip("rescaling only for variational_gamma")
        ts = self.simulate(10, seed=2)
        times, frac = PLACEMENTS[placement]
        before = [int(f * ts.num_nodes) for f in frac]
        tables, iso, old_to_new = insert_isolated_nodes(ts, times, before)
        ts_iso = tables.tree_sequence()
        func = METHODS[method]
        kw = dict(extra)
        if method == "variational_gamma":
            dated, fit = func(ts, return_fit=True, **kw)
            dated_iso, fit_iso = func(ts_iso, return_fit=True, **kw)
        else:
            dated, dated_iso = func(ts, **kw), func(ts_iso, **kw)

        # isolated nodes are untouched
        assert np.array_equal(dated_iso.nodes_time[iso], ts_iso.nodes_time[iso])
        assert np.array_equal(np.sort(dated_iso.nodes_time[iso]), np.sort(times))
        assert not np.any(dated_iso.nodes_flags[iso] & tskit.NODE_IS_SAMPLE)
        # everything else is identical
        assert np.allclose(dated.nodes_time, dated_iso.nodes_time[old_to_new])
        assert np.allclose(dated.mutations_time, dated_iso.mutations_time)
        assert dated_iso.num_edges == dated.num_edges

        if method != "maximization":  # maximization sets no posterior metadata
            mn = np.array([n.metadata["mn"] for n in dated_iso.nodes()])
            vr = np.array([n.metadata["vr"] for n in dated_iso.nodes()])
            mn0 = np.array([n.metadata["mn"] for n in dated.nodes()])
            vr0 = np.array([n.metadata["vr"] for n in dated.nodes()])
            assert np.all(np.isnan(mn[iso])) and np.all(np.isnan(vr[iso]))
            assert np.allclose(mn[old_to_new], mn0)
            assert np.allclose(vr[old_to_new], vr0)
        if method == "variational_gamma":
            mm = [m.metadata["mn"] for m in dated_iso.mutations()]
            mm0 = [m.metadata["mn"] for m in dated.mutations()]
            assert np.allclose(mm, mm0)
            p, p0 = fit_iso.node_posteriors(), fit.node_posteriors()
            for field in ("mean", "variance"):
                assert np.all(np.isnan(p[field][iso]))
                assert np.allclose(p[field][old_to_new], p0[field])

    @pytest.mark.parametrize("method", METHODS)
    def test_mutation_on_isolated_node(self, method):
        ts = self.simulate(10, seed=3)
        tables, iso, old_to_new = insert_isolated_nodes(ts, [5.0], [ts.num_nodes])
        # new site with a single mutation above the isolated node
        site = tables.sites.add_row(
            position=ts.sequence_length - 0.5, ancestral_state="A"
        )
        tables.mutations.add_row(site=site, node=iso[0], derived_state="T")
        tables.sort()
        ts_iso = tables.tree_sequence()
        func = METHODS[method]
        dated, dated_iso = func(ts), func(ts_iso)
        assert np.allclose(dated.nodes_time, dated_iso.nodes_time[old_to_new])
        # mutation on the isolated node is dated at the node's (fixed) time
        assert dated_iso.mutations_time[-1] == 5.0

@nspope nspope left a comment

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks good materially, but I think we need a bit more testing and also want to make sure isolated samples can't creep through to the discrete-time methods.

Comment thread CHANGELOG.md
- Isolated non-sample nodes (those in no edges) are now allowed by all dating
methods. They cannot be dated, so keep their input times (with NaN
posterior mean and variance in their metadata) and do not affect the dating of
other nodes. Isolated samples still raise an error.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think isolated samples raise an error for the discrete-grid methods? In fact, they will change the results because they alter the prior. I think we should disallow across the board.

Here's a sketch of a test:

import msprime
import numpy as np
import pytest
import tskit

import tsdate

METHODS = {
    "variational_gamma": lambda ts: tsdate.variational_gamma(ts, mutation_rate=1e-8),
    "inside_outside": lambda ts: tsdate.inside_outside(
        ts, mutation_rate=1e-8, population_size=1e4
    ),
    "maximization": lambda ts: tsdate.maximization(
        ts, mutation_rate=1e-8, population_size=1e4
    ),
}


def simulate():
    ts = msprime.sim_ancestry(
        10,
        sequence_length=2e5,
        recombination_rate=1e-8,
        population_size=1e4,
        random_seed=1,
    )
    return msprime.sim_mutations(ts, rate=1e-8, random_seed=1)


def with_isolated_sample(ts, time=0.0):
    tables = ts.dump_tables()
    tables.nodes.add_row(flags=tskit.NODE_IS_SAMPLE, time=time)
    return tables.tree_sequence()


class TestIsolatedSamplesProhibited:
    @pytest.mark.parametrize("method", METHODS)
    def test_isolated_sample_raises(self, method):
        ts_iso = with_isolated_sample(simulate())
        with pytest.raises(ValueError, match="disconnected"):
            METHODS[method](ts_iso)

This branch has not been deployed

No deployments
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Labels

None yet

Projects

None yet

Development

Successfully merging this pull request may close these issues.

2 participants