From 4a2facecf11a84bf0df43cbd4f6a1526d0903017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9cile=20An=C3=A9?= Date: Tue, 4 Jun 2024 13:02:00 -0500 Subject: [PATCH] documentation; small fix; addleaf! returns leaf; ready for v0.16.4 (#212) fix: r2 corrected in ftest addleaf! output new leaf node, not net: breaking change, but internal function documentation: - workflow - badges - workshop link - ftest warnings explained - typo fix in doc for slurm --- .github/workflows/ci.yml | 51 ++++++++--------------------- .github/workflows/doccleanup.yml | 35 +++++++++++--------- .github/workflows/documentation.yml | 30 +++++++++++++++++ Project.toml | 2 +- README.md | 15 ++++++--- docs/make.jl | 7 +++- docs/readme.md | 4 +-- docs/src/index.md | 4 ++- docs/src/man/netmanipulation.md | 5 +-- docs/src/man/snaq_plot.md | 2 +- docs/src/man/trait_tree.md | 16 ++++++++- src/manipulateNet.jl | 9 ++--- src/traits.jl | 18 ++++++++-- test/test_lm.jl | 14 ++++---- 14 files changed, 134 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/documentation.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fff98e89..df9c0a7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,13 @@ on: push: branches: - master - tags: '*' + tags: ['*'] + +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} jobs: test: @@ -17,7 +23,7 @@ jobs: fail-fast: false matrix: version: - - '1' # will expand to the latest stable 1.x release of Julia. + - '1' os: - ubuntu-latest - macos-latest @@ -25,45 +31,16 @@ jobs: arch: - x64 steps: - - uses: actions/checkout@v2 - - uses: julia-actions/setup-julia@v1 + - uses: actions/checkout@v4 + - uses: julia-actions/setup-julia@latest with: version: ${{ matrix.version }} arch: ${{ matrix.arch }} - - uses: actions/cache@v1 - env: - cache-name: cache-artifacts - with: - path: ~/.julia/artifacts - key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} - restore-keys: | - ${{ runner.os }}-test-${{ env.cache-name }}- - ${{ runner.os }}-test- - ${{ runner.os }}- + - uses: julia-actions/cache@v2 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v2 - with: - file: lcov.info - docs: - name: Documentation - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: r-lib/actions/setup-r@v2 - - run: echo "LD_LIBRARY_PATH=$(R RHOME)/lib" >> $GITHUB_ENV - - uses: julia-actions/setup-julia@v1 + - uses: codecov/codecov-action@v4 with: - version: '1' - - name: install dependencies - run: | - julia --project=docs -e ' - using Pkg - Pkg.develop(PackageSpec(path=pwd())) - Pkg.instantiate()' - - name: make docs - run: julia --project=docs docs/make.jl - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/doccleanup.yml b/.github/workflows/doccleanup.yml index 15680241..2d2b297d 100644 --- a/.github/workflows/doccleanup.yml +++ b/.github/workflows/doccleanup.yml @@ -1,28 +1,33 @@ -name: Doc Preview Cleanup +name: doc preview cleanup on: pull_request: types: [closed] +# ensure that only one "doc preview cleanup" workflow is force-pushing at a time +concurrency: + group: doc-preview-cleanup + cancel-in-progress: false + jobs: doc-preview-cleanup: runs-on: ubuntu-latest + permissions: + contents: write steps: - - name: Checkout gh-pages branch - uses: actions/checkout@v2 + - name: checkout gh-pages branch + uses: actions/checkout@v4 with: ref: gh-pages - - - name: Delete preview and history + - name: delete preview and history + push changes run: | - git config user.name "Documenter.jl" - git config user.email "documenter@juliadocs.github.io" - git rm -rf "previews/PR$PRNUM" - git commit -m "delete preview" - git branch gh-pages-new $(echo "delete history" | git commit-tree HEAD^{tree}) + if [ -d "${preview_dir}" ]; then + git config user.name "Documenter.jl" + git config user.email "documenter@juliadocs.github.io" + git rm -rf "${preview_dir}" + git commit -m "delete preview" + git branch gh-pages-new $(echo "delete history" | git commit-tree HEAD^{tree}) + git push --force origin gh-pages-new:gh-pages + fi env: - PRNUM: ${{ github.event.number }} - - - name: Push changes - run: | - git push --force origin gh-pages-new:gh-pages + preview_dir: previews/PR${{ github.event.number }} diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 00000000..7e8c1264 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,30 @@ +name: Documenter + +on: + push: + branches: + - main + tags: '*' + pull_request: + +jobs: + Documenter: + name: documentation + permissions: + contents: write + pull-requests: read + statuses: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: r-lib/actions/setup-r@v2 + - run: echo "LD_LIBRARY_PATH=$(R RHOME)/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + - uses: julia-actions/setup-julia@v2 + with: + version: '1' + - uses: julia-actions/cache@v2 + - uses: julia-actions/julia-buildpkg@latest + - uses: julia-actions/julia-docdeploy@latest + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} diff --git a/Project.toml b/Project.toml index 51f36bf0..15ef386d 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "PhyloNetworks" uuid = "33ad39ac-ed31-50eb-9b15-43d0656eaa72" license = "MIT" -version = "0.16.3" +version = "0.16.4" [deps] BioSequences = "7e6ae17a-c86d-528c-b3b9-7f778a29fe59" diff --git a/README.md b/README.md index 95d56c20..c18d30bd 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,12 @@ # PhyloNetworks: analysis for phylogenetic networks +[![doc stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://crsl4.github.io/PhyloNetworks.jl/stable) +[![doc dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://crsl4.github.io/PhyloNetworks.jl/dev) [![Build status](https://github.com/crsl4/PhyloNetworks.jl/workflows/CI/badge.svg?branch=master)](https://github.com/crsl4/PhyloNetworks.jl/actions/workflows/ci.yml) -[![](https://img.shields.io/badge/docs-stable-blue.svg)](https://crsl4.github.io/PhyloNetworks.jl/stable) -[![](https://img.shields.io/badge/docs-dev-blue.svg)](https://crsl4.github.io/PhyloNetworks.jl/dev) -[![codecov](https://codecov.io/gh/crsl4/PhyloNetworks.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/crsl4/PhyloNetworks.jl) +[![coverage](https://codecov.io/gh/crsl4/PhyloNetworks.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/crsl4/PhyloNetworks.jl) +[![Code Style: Blue](https://img.shields.io/badge/code%20style-blue-4495d1.svg)](https://github.com/invenia/BlueStyle) +[![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor's%20Guide-blueviolet)](https://github.com/SciML/ColPrac) +[![PkgEval](https://JuliaCI.github.io/NanosoldierReports/pkgeval_badges/P/PhyloNetworks.svg)](https://JuliaCI.github.io/NanosoldierReports/pkgeval_badges/report.html) ## Overview @@ -18,7 +21,7 @@ PhyloNetworks is a [Julia](http://julialang.org) package with utilities to: (Robinson-Foulds distance on trees) - summarize samples of bootstrap networks (or trees) with edge and node support -- estimate species networks from multilocus data (see below) +- estimate species networks from multilocus data: SNaQ - phylogenetic comparative methods for continuous trait evolution on species networks / trees - plot networks (and trees), via the companion package @@ -28,7 +31,9 @@ To get help, check - the [latest documentation](https://crsl4.github.io/PhyloNetworks.jl/dev) - the [wiki](https://github.com/crsl4/PhyloNetworks.jl/wiki) for a step-by-step tutorial - (July 2018) with background on networks + with background on networks (last revised 2022) +- [tutorial](https://cecileane.github.io/networkPCM-workshop/) for + comparative methods, including network calibration (2023 workshop) - the [google group](https://groups.google.com/forum/#!forum/phylonetworks-users) for common questions. Join the group to post/email your questions, or to receive information on new versions, bugs fixed, etc. diff --git a/docs/make.jl b/docs/make.jl index 5c0fac64..44b5341d 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -13,7 +13,11 @@ makedocs( modules = [PhyloNetworks], # to list methods from PhyloNetworks only, not from Base etc. format = Documenter.HTML( prettyurls = get(ENV, "CI", nothing) == "true", # easier local build - size_threshold = 600 * 2^10, size_threshold_warn = 500 * 2^10), # 600 KiB + size_threshold = 600 * 2^10, + size_threshold_warn = 500 * 2^10, # 600 KiB + canonical="https://crsl4.github.io/PhyloNetworks.jl/stable/", + edit_link="master", + ), # exception, so warning-only for :missing_docs. List all others: warnonly = Documenter.except(:autodocs_block, :cross_references, :docs_block, :doctest, :eval_block, :example_block, :footnote, :linkcheck_remotes, @@ -46,4 +50,5 @@ makedocs( deploydocs( repo = "github.com/crsl4/PhyloNetworks.jl.git", push_preview = true, + devbranch = "master", ) diff --git a/docs/readme.md b/docs/readme.md index 7d4b40b3..c9ea3c58 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -1,13 +1,13 @@ # notes to maintain documentation -- built with [Documenter](https://juliadocs.github.io/Documenter.jl). +- built with [Documenter](https://documenter.juliadocs.org/stable/). - deployed [here](https://crsl4.github.io/PhyloNetworks.jl/) (go to `dev/` or `stable/`) using GitHub and files committed to the `gh-pages` branch. ## how it works: overview -- `.github/workflows/ci.yml` asks to start the doc project +- `.github/workflows/documentation.yml` asks to start the doc project (installs R and dependencies like `PhyloPlots` & `Documenter`) and run `./docs/make.jl` after a successful test & build. - the julia script `docs/make.jl` has these steps: diff --git a/docs/src/index.md b/docs/src/index.md index 920bec01..cbe67101 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -10,8 +10,10 @@ and their use for trait evolution. **How to get help** - the package [wiki](https://github.com/crsl4/PhyloNetworks.jl/wiki) has a step-by-step - tutorial, done for the 2019 MBL workshop, with background on networks and + tutorial, done for the MBL workshop (last revised 2022), with background on networks and explanations. +- [tutorial](https://cecileane.github.io/networkPCM-workshop/) for + comparative methods, including network calibration (2023 workshop) - the [google group](https://groups.google.com/forum/#!forum/phylonetworks-users) has answers to common questions. - the [Manual](@ref) below has a quick tutorial (navigation on the left). diff --git a/docs/src/man/netmanipulation.md b/docs/src/man/netmanipulation.md index 376ec788..8451c82b 100644 --- a/docs/src/man/netmanipulation.md +++ b/docs/src/man/netmanipulation.md @@ -36,9 +36,10 @@ To modify a network, for example: [`deleteaboveLSA!`](@ref) (the "least stable ancestor" may be different from the root) - [`deleteHybridThreshold!`](@ref) to simplify a network by deleting edges with small γ's - [`removedegree2nodes!`](@ref), [`shrink3cycles!`](@ref), [`shrink2cycles!`](@ref), - [`PhyloNetworks.shrinkedge!`](@ref)] + [`PhyloNetworks.shrinkedge!`](@ref) - [`PhyloNetworks.addleaf!`](@ref), - [`PhyloNetworks.deletehybridedge!`](@ref), [`PhyloNetworks.addhybridedge!`](@ref) + [`PhyloNetworks.deletehybridedge!`](@ref), + [`PhyloNetworks.addhybridedge!`](@ref) - [`nni!`](@ref) (nearest neighbor interchange), [`PhyloNetworks.fliphybrid!`](@ref) to flip the direction of a hybrid edge - [`PhyloNetworks.unzip_canonical!`](@ref) to "unzip" (or zip down) all diff --git a/docs/src/man/snaq_plot.md b/docs/src/man/snaq_plot.md index 078631c3..e02658c6 100644 --- a/docs/src/man/snaq_plot.md +++ b/docs/src/man/snaq_plot.md @@ -261,7 +261,7 @@ Then log out of the cluster and go for coffee. echo "slurm task ID = $SLURM_ARRAY_TASK_ID used as hmax" echo "start of SNaQ parallel runs on $(hostname)" # finally: launch the julia script, using Julia executable appropriate for slurm, with full paths: -/workspace/software/bin/julia --history-file=no -- runSNaQ.jl $SLURM_ARRAY_TASK_ID 30 > net$SLURM_ARRAY_TASK_ID_30runs.screenlog 2>&1 +/workspace/software/bin/julia --history-file=no -- runSNaQ.jl $SLURM_ARRAY_TASK_ID 30 > net${SLURM_ARRAY_TASK_ID}_30runs.screenlog 2>&1 echo "end of SNaQ run ..." ``` diff --git a/docs/src/man/trait_tree.md b/docs/src/man/trait_tree.md index 11aefbb2..ac1c73ca 100644 --- a/docs/src/man/trait_tree.md +++ b/docs/src/man/trait_tree.md @@ -508,10 +508,24 @@ complex cases, it is possible to do a Fisher F test, thanks to the `GLM` function `ftest`. ```@example tree_trait fit_null = phylolm(@formula(trait ~ 1), dat, truenet) # fit against the null (no shift) -ftest(fit_sh, fit_null) # nested models +ftest(fit_null, fit_sh) # nested models ``` Here, this test is equivalent to the Fisher F test, and gives the same p-value. +!!! note "Warnings from GLM" + A warning may appear, saying + "Starting from GLM.jl 1.8, null model is defined as having no predictor at all when a model without an intercept is passed." + - Why? `ftest` is inherited from the GLM package, which does not know that + the intercept term is not a column of ones after transformation to remove + the phylogenetic correlation. This is why `ftest` sends a warning for + each model, when multiple models are compared. + - These specific warnings can be ignored: + * F values and p-values are correct + * R² values are also correct: they are obtained with the + `r2` function for phylogenetic linear models. + A future version of the package will attempt to remove these warnings + specifically. + Note that models need to be ordered by complexity, when given to `ftest`: either from most complex to most simple, or from most simple to most complex. In the output table, models are listed in the order in which they were given. diff --git a/src/manipulateNet.jl b/src/manipulateNet.jl index 0b4529ca..5c5e1a3d 100644 --- a/src/manipulateNet.jl +++ b/src/manipulateNet.jl @@ -567,6 +567,8 @@ Add a new external edge between `node` or between the "middle" of `edge` and a newly-created leaf, of name `leafname`. By default, the new edge length is missing (-1). +output: newly created leaf node. + # examples ```jldoctest @@ -616,13 +618,12 @@ function addleaf!(net::HybridNetwork, speciesnode::Node, leafname::String, edgel net.numTaxa -= 1 end pushNode!(net, newleaf) # push node into network (see auxillary.jl) - return net + return newleaf end function addleaf!(net::HybridNetwork, startingedge::Edge, leafname::String, edgelength::Float64=-1.0) - newnode, newedge = breakedge!(startingedge, net) - addleaf!(net, newnode, leafname, edgelength) - return net + newnode, _ = breakedge!(startingedge, net) + return addleaf!(net, newnode, leafname, edgelength) end diff --git a/src/traits.jl b/src/traits.jl index f25718e9..af183d04 100644 --- a/src/traits.jl +++ b/src/traits.jl @@ -3019,14 +3019,28 @@ isnested(::Union{PagelLambda,ScalingHybrid}, ::BM) = false isnested(::ScalingHybrid,::PagelLambda) = false isnested(::PagelLambda,::ScalingHybrid) = false -## ANOVA using ftest from GLM - need version 0.8.1 +#= ANOVA using ftest from GLM - need version 0.8.1 + As of GLM v1.8, ftest throws a warning on typical BM models, one per model: + "Starting from GLM.jl 1.8, null model is defined as having no predictor at all when a model without an intercept is passed." + This is because after transforming the data to de-correlate the residuals, + the transformed intercept vector is not proportional to the constant vector 1. + The warning is from: ftest → r2(phylomodel.lm) → nulldeviance(phylomodel.lm) → warning. + R² by GLM is wrong: assume *no* intercept, and are based on the transformed data. + R² corrected them below: r2(phylomodel) reimplemented here. + But nulldeviance(phylomodel) does *not* call nulldeviance(phylomodel.lm), + instead re-implemented here to use the intercept properly. + Keep the warnings: unless they can be suppressed with specificity + Ideally: modify `ftest` here or in GLM. +=# function GLM.ftest(objs::PhyloNetworkLinearModel...) if !all( isa(o.evomodel,BM) && isnothing(o.model_within) for o in objs) throw(ArgumentError("""F test is only valid for the vanilla BM model. Use a likelihood ratio test instead with function `lrtest`.""")) end objslm = [obj.lm for obj in objs] - return ftest(objslm...) + resGLM = ftest(objslm...) + resGLM.r2 = r2.(objs) + return resGLM end ## ANOVA: old version - kept for tests purposes - do not export """ diff --git a/test/test_lm.jl b/test/test_lm.jl index 806321ae..b3d1c56b 100644 --- a/test/test_lm.jl +++ b/test/test_lm.jl @@ -210,7 +210,13 @@ modnull = phylolm(@formula(trait ~ 1), dfr, net) @test sigma2_phylo(modnull) ≈ 0.6517876326943942 atol=1e-6 # using REML modhom = phylolm(@formula(trait ~ sum), dfr, net) modhet = phylolm(@formula(trait ~ sum + shift_8), dfr, net) -table1 = ftest(modhet, modhom, modnull) +#= 3 warnings thrown by ftest, one for each model, because after transforming the + data to de-correlate the results, the intercept vector is not ∝ 1. + Keep the warnings: because incorrect R² values in the ftest output +=# +table1 = redirect_stdio(stderr=devnull) do # to avoid seeing the warnings + ftest(modhet, modhom, modnull) +end table2 = PhyloNetworks.anova(modnull, modhom, modhet) @test table1.fstat[2] ≈ table2[2,:F] @@ -218,11 +224,7 @@ table2 = PhyloNetworks.anova(modnull, modhom, modhet) @test table1.pval[2] ≈ table2[2,Symbol("Pr(>F)")] @test table1.pval[3] ≈ table2[1,Symbol("Pr(>F)")] @test hasintercept(modnull) && hasintercept(modhom) && hasintercept(modhet) -# ## Replace next 4 lines with previous ones when GLM.ftest available -# @test table1[:F][2] ≈ table2[:F][2] -# @test table1[:F][1] ≈ table2[:F][1] -# @test table1[Symbol("Pr(>F)")][1] ≈ table2[Symbol("Pr(>F)")][1] -# @test table1[Symbol("Pr(>F)")][2] ≈ table2[Symbol("Pr(>F)")][2] +@test all(isapprox.(table1.r2, (0.8398130376214782, 0.006032952123011026, 0), atol=1e-15)) # Check that it is the same as doing shift_8 + shift_17 modhetbis = phylolm(@formula(trait ~ shift_8 + shift_17), dfr, net)