Merge pull request #8 from flyconnectome/banc

basic support for the BANC

.Rbuildignore

@@ -8,3 +8,4 @@
 ^spare$
 ^vignettes/articles$
 
+^data-raw$

.github/workflows/R-CMD-check.yaml

@@ -49,7 +49,9 @@ jobs:
         run: |
           pak::local_install()
           library(fafbseg)
-          simple_python()
+          simple_python(pkgs = 'fafbseg')
+          simple_python('none', pkgs = 'caveclient==5.15.2')
+          simple_python('none', pkgs = 'pyarrow==12.0.1')
           dr_fafbseg()
         shell: Rscript {0}
 

.github/workflows/pkgdown.yaml

@@ -0,0 +1,69 @@
+# Workflow derived from https://github.com/r-lib/actions/tree/v2/examples
+# Need help debugging build failures? Start at https://github.com/r-lib/actions#where-to-find-help
+on:
+  push:
+    branches: [main, master]
+  pull_request:
+    branches: [main, master]
+  release:
+    types: [published]
+  workflow_dispatch:
+
+name: pkgdown
+
+permissions: read-all
+
+jobs:
+  pkgdown:
+    runs-on: ubuntu-latest
+    # Only restrict concurrency for non-PR jobs
+    concurrency:
+      group: pkgdown-${{ github.event_name != 'pull_request' || github.run_id }}
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+      CHUNKEDGRAPH_SECRET: ${{ secrets.CHUNKEDGRAPH_SECRET }}
+      FLYTABLE_TOKEN: ${{ secrets.FLYTABLE_TOKEN }}
+      RETICULATE_MINICONDA_PYTHON_VERSION: 3.8
+      RGL_USE_NULL: TRUE
+
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: r-lib/actions/setup-pandoc@v2
+
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          use-public-rspm: true
+
+      - uses: r-lib/actions/setup-r-dependencies@v2
+        with:
+          extra-packages: any::pkgdown, catmaid=natverse/rcatmaid, local::.
+          needs: website
+
+      - name: Install fafbseg + python
+        run: |
+          pak::local_install()
+          library(fafbseg)
+          simple_python(pkgs = 'fafbseg')
+          # simple_python('none', pkgs = 'caveclient==5.15.2')
+          # simple_python('none', pkgs = 'pyarrow==12.0.1')
+          dr_fafbseg()
+        shell: Rscript {0}
+
+      - name: writetoken
+        run: fafbseg::flywire_set_token(token=Sys.getenv("CHUNKEDGRAPH_SECRET"))
+        shell: Rscript {0}
+
+      - name: Build site
+        run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE)
+        shell: Rscript {0}
+
+      - name: Deploy to GitHub pages 🚀
+        if: github.event_name != 'pull_request'
+        uses: JamesIves/github-pages-deploy-action@v4.5.0
+        with:
+          clean: false
+          branch: gh-pages
+          folder: docs

.gitignore

@@ -3,3 +3,4 @@
 .RData
 .Ruserdata
 docs
+inst/doc

DESCRIPTION

@@ -12,11 +12,11 @@ Description: Access to the FANC EM dataset, focussing on support for
     autosegmentation. Organised as a thin wrapper around the fafbseg
     package.
 License: GPL-3
-URL: https://github.com/flyconnectome/fancr
+URL: https://github.com/flyconnectome/fancr, https://flyconnectome.github.io/fancr/
 BugReports: https://github.com/flyconnectome/fancr/issues
 Depends: 
     R (>= 2.10),
-    fafbseg (>= 0.13.0.9100),
+    fafbseg (> 0.14.1),
     nat.flybrains
 Imports: 
     nat,
@@ -26,7 +26,8 @@ Imports:
     jsonlite,
     pbapply,
     dplyr,
-    checkmate
+    checkmate,
+    utils
 Suggests: 
     testthat (>= 3.0.0),
     reticulate,
@@ -35,15 +36,23 @@ Suggests:
     usethis,
     rmarkdown,
     spelling,
-    arrow
+    arrow,
+    knitr,
+    nat.nblast,
+    Morpho
 Remotes: 
     natverse/nat,
+    natverse/nat.nblast,
     natverse/fafbseg,
     natverse/nat.flybrains,
     natverse/nat.templatebrains
 Config/testthat/edition: 3
+Config/Needs/website: 
+    knitr,
+    nat.nblast
 Encoding: UTF-8
 Language: en-GB
 LazyData: true
 Roxygen: list(markdown = TRUE)
 RoxygenNote: 7.3.1
+VignetteBuilder: knitr

NAMESPACE

@@ -1,5 +1,12 @@
 # Generated by roxygen2: do not edit by hand
 
+export(banc_cave_client)
+export(banc_cave_query)
+export(banc_lr_position)
+export(banc_nm2raw)
+export(banc_raw2nm)
+export(banc_voxdims)
+export(choose_banc)
 export(choose_fanc)
 export(dr_fanc)
 export(fanc4to3)
@@ -23,15 +30,24 @@ export(fanc_segid_from_cellid)
 export(fanc_set_token)
 export(fanc_voxdims)
 export(fanc_xyz2id)
+export(mirror_banc)
 export(read_fanc_meshes)
 export(transform_fanc2manc)
+export(with_banc)
 export(with_fanc)
 import(bit64)
 import(fafbseg)
 import(nat.flybrains)
+importFrom(dplyr,group_by)
+importFrom(dplyr,summarise)
+importFrom(nat,"xyzmatrix<-")
+importFrom(nat,is.neuronlist)
+importFrom(nat,nvertices)
 importFrom(nat,tpsreg)
 importFrom(nat,xform)
 importFrom(nat,xyzmatrix)
+importFrom(nat.templatebrains,mirror_brain)
+importFrom(nat.templatebrains,templatebrain)
 importFrom(pbapply,pbsapply)
 importFrom(utils,browseURL)
 importFrom(utils,read.csv)

R/cave.R

@@ -12,13 +12,14 @@
 #' @seealso \code{\link[fafbseg]{flywire_cave_query}}
 #' @examples
 #' \donttest{
-#' nuclei_aug2021ver2=fanc_cave_query(table = "nuclei_aug2021ver2")
+#' #nucleus_mar2022=fanc_cave_query(table = "nucleus_mar2022")
+#' neuron_somas_dec2022=fanc_cave_query(table = "neuron_somas_dec2022")
 #' head(neuron_somas_dec2022)
 #' hist(neuron_somas_dec2022$volume)
 #' hist(neuron_somas_dec2022$volume^(1/3))
 #' }
 #' \dontrun{
-#' points3d(xyzmatrix(nuclei_aug2021ver2$pt_position))
+#' points3d(fanc_raw2nm(neuron_somas_dec2022$pt_position), col='red')
 #' }
 fanc_cave_query <- function(table, datastack_name = NULL, live=TRUE, ...) {
   if(is.null(datastack_name)) datastack_name=fanc_datastack_name()
@@ -37,6 +38,27 @@ fanc_cave_query <- function(table, datastack_name = NULL, live=TRUE, ...) {
 #' fcc$materialize$get_table_metadata(tables[1])
 #' }
 fanc_cave_client <- function() {
-  with_fanc(flywire_cave_client())
+  with_fanc(flywire_cave_client(), force = FALSE)
+}
+
+#' @rdname fanc_cave_client
+#' @export
+banc_cave_client <- function() {
+  with_banc(flywire_cave_client())
+}
+
+#' @rdname fanc_cave_query
+#' @export
+#' @examples
+#' \dontrun{
+#' library(dplyr)
+#' cell_info=banc_cave_query('cell_info')
+#' cell_info %>%
+#'   filter(tag2=='anterior-posterior projection pattern') %>%
+#'   count(tag)
+#' }
+banc_cave_query <- function(table, datastack_name = NULL, live=TRUE, ...) {
+  if(is.null(datastack_name)) datastack_name=banc_datastack_name()
+  fafbseg::flywire_cave_query(table = table, datastack_name = datastack_name, live=live, ...)
 }
 

R/cloudvolume.R

@@ -4,7 +4,8 @@ fanc_cloudvolume <- function(...) {
 }
 
 fanc_cloudvolume_url <- function() {
-  with_fanc(getOption("fafbseg.cloudvolume.url"))
+  rr=with_fanc(getOption("fafbseg.cloudvolume.url"), force = FALSE)
+  sub("graphene://middleauth+", "graphene://", rr, fixed = TRUE)
 }
 
 fanc_api_url <- function(endpoint="") {
@@ -72,6 +73,6 @@ fanc_api_report <- function() {
     print(ff)
   }
 
-  u=with_fanc(fafbseg:::check_cloudvolume_url(set = F))
+  u=with_fanc(fafbseg:::check_cloudvolume_url(set = F), force = FALSE)
   cat("\nZetta cloudvolume URL:", u)
 }

R/coords.R

@@ -17,6 +17,16 @@ fanc_voxdims <- memoise::memoise(function(url=choose_fanc(set=FALSE)[['fafbseg.s
 })
 
 
+#' @description \code{banc_voxdims} returns the image voxel dimensions for the
+#'   BANC which are normally used to scale between raw and nm coordinates.
+#'
+#' @rdname fanc_voxdims
+#' @export
+banc_voxdims <- memoise::memoise(function(url=choose_banc(set=FALSE)[['fafbseg.sampleurl']]) {
+  fafbseg::flywire_voxdims(url)
+})
+
+
 #' @param x 3D coordinates in any form compatible with \code{\link{xyzmatrix}}
 #'
 #' @return for \code{fanc_raw2nm} and \code{fanc_nm2raw} an Nx3 matrix of
@@ -49,3 +59,12 @@ fanc_raw2nm <- function(x, vd=fanc_voxdims()) {
   xyz[,3]=xyz[,3]*vd[3]
   xyz
 }
+
+#' @export
+#' @rdname fanc_voxdims
+banc_raw2nm <- function(x, vd=banc_voxdims()) fanc_raw2nm(x, vd=vd)
+
+#' @export
+#' @rdname fanc_voxdims
+banc_nm2raw <- function(x, vd=banc_voxdims()) fanc_nm2raw(x, vd=vd)
+

R/data.R

@@ -2,7 +2,8 @@
 #'
 #' @name FANC.surf
 #' @docType data
-#' @description This is unsymmetrical and not normalized version of the mesh.
+#' @description This is unsymmetrical and not normalized version of the mesh. It
+#'   is calibrated in nm.
 #'
 #' @examples
 #' \dontrun{
@@ -11,3 +12,48 @@
 #' wire3d(FANC.surf)
 #' }
 "FANC.surf"
+
+
+#' Simplified tissue surface of BANC
+#'
+#' @name BANC.surf
+#' @docType data
+#' @description This is unsymmetrical and not normalized version of the mesh. It
+#'   is calibrated in nm. See \code{data-raw/BANC.R} for details of how it was
+#'   generated.
+#'
+#' @examples
+#' \dontrun{
+#' # Depends on nat
+#' library(nat)
+#' wire3d(BANC.surf, col='grey')
+#' }
+"BANC.surf"
+
+#' Provisional template brain object for BANC
+#'
+#' @name BANC
+#' @docType data
+#' @description This is calibrated in nm. See \code{data-raw/BANC.R} for details
+#'   of how it was generated. It is still provisional pending synpase
+#'   cloud-based registrations.
+#'
+#' @examples
+#' BANC
+#' # Depends on nat
+#' library(nat)
+#' boundingbox(BANC)
+"BANC"
+
+#' Provisional mirroring registration for BANC (lanmarks+thin plate splines)
+#'
+#' @name mirror_banc_lm
+#' @docType data
+#' @description This is calibrated in nm. See \code{data-raw/mirror_banc.R} for
+#'   details of how it was generated. It is still provisional pending synpase
+#'   cloud-based registrations.
+#'
+#' @examples
+#' utils::str(mirror_banc_lm)
+"mirror_banc_lm"
+

R/ids.R

@@ -105,7 +105,7 @@ fanc_supervoxels <- function(x, voxdims=c(4.3,4.3,45)) {
 #' @examples
 #' fanc_islatest("648518346473954669")
 fanc_islatest <- function(x, timestamp=NULL, ...) {
-  with_fanc(flywire_islatest(x=x, timestamp = timestamp, ...))
+  with_fanc(flywire_islatest(x=x, timestamp = timestamp, ...), force = FALSE)
 }
 
 
@@ -122,7 +122,7 @@ fanc_islatest <- function(x, timestamp=NULL, ...) {
 #' fanc_latestid("648518346473954669")
 #' }
 fanc_latestid <- function(rootid, sample=1000L, cloudvolume.url=NULL, Verbose=FALSE, ...) {
-  with_fanc(flywire_latestid(rootid=rootid, sample = sample, Verbose=Verbose, ...))
+  with_fanc(flywire_latestid(rootid=rootid, sample = sample, Verbose=Verbose, ...), force = FALSE)
 }
 
 
@@ -207,8 +207,13 @@ fanc_ids <- function(x, integer64=NA) {
 #' }
 fanc_cellid_from_segid <- function(rootids=NULL, timestamp=NULL, version=NULL, cellid_table = NULL, rval=c("ids", 'data.frame')) {
   rval=match.arg(rval)
-  if(is.null(cellid_table))
-    cellid_table=fanc_cellid_table()
+  use_banc <- getOption("fancr.use_banc", default = FALSE)
+
+  if(is.null(cellid_table)) {
+    cellid_table=if(use_banc) fanc_cellid_table(fac = banc_cave_client())
+    else fanc_cellid_table()
+  }
+
   if(!is.null(rootids)) {
     rootids=fanc_ids(rootids, integer64=F)
   idlist=list(pt_root_id=rootids)
@@ -240,8 +245,13 @@ fanc_cellid_from_segid <- function(rootids=NULL, timestamp=NULL, version=NULL, c
 #' }
 fanc_segid_from_cellid <- function(cellids=NULL, timestamp=NULL, version=NULL, rval=c("ids", 'data.frame'), integer64=FALSE, cellid_table = NULL) {
   rval=match.arg(rval)
-  if(is.null(cellid_table))
-    cellid_table=fanc_cellid_table()
+  use_banc <- getOption("fancr.use_banc", default = FALSE)
+
+  if(is.null(cellid_table)) {
+    cellid_table=if(use_banc) fanc_cellid_table(fac = banc_cave_client())
+    else fanc_cellid_table()
+  }
+
   if(!is.null(cellids)) {
     cellids <- checkmate::assert_integerish(cellids, coerce = T)
     idlist=list(id=cellids)

R/l2.R

@@ -52,7 +52,7 @@
 fanc_read_l2dp <- function(id, OmitFailures=TRUE, dataset=NULL, ...) {
   id=fanc_ids(id)
   if(is.null(dataset))
-    dataset=with_fanc(getOption("fafbseg.cave.datastack_name"))
+    dataset=with_fanc(getOption("fafbseg.cave.datastack_name"), force = FALSE)
   fafbseg::read_l2dp(id, dataset=dataset, OmitFailures=OmitFailures, ...)
 }
 
@@ -65,7 +65,7 @@ fanc_read_l2skel <- function(id, OmitFailures=TRUE, dataset=NULL, ...) {
 
   if("set_default_dataset" %in% names(fp$flywire)) {
     # new fafbseg-py, everything is simpler
-    return(with_fanc(fafbseg::read_l2skel(id)))
+    return(with_fanc(fafbseg::read_l2skel(id), force = FALSE))
   }
   # this used to work with FANC and older fafbseg-py, but not sure if it still works ...
   # manually set the cloudvolume url / cave datastack name