Merge pull request #1058 from seeM/fix-sync

fix `nbdev_update`

Author: jph00

nbdev/_modidx.py

@@ -57,6 +57,7 @@
                                 'nbdev.doclinks._find_mod': ('api/doclinks.html#_find_mod', 'nbdev/doclinks.py'),
                                 'nbdev.doclinks._get_exps': ('api/doclinks.html#_get_exps', 'nbdev/doclinks.py'),
                                 'nbdev.doclinks._get_modidx': ('api/doclinks.html#_get_modidx', 'nbdev/doclinks.py'),
+                                'nbdev.doclinks._iter_py_cells': ('api/doclinks.html#_iter_py_cells', 'nbdev/doclinks.py'),
                                 'nbdev.doclinks._lineno': ('api/doclinks.html#_lineno', 'nbdev/doclinks.py'),
                                 'nbdev.doclinks._nbpath2html': ('api/doclinks.html#_nbpath2html', 'nbdev/doclinks.py'),
                                 'nbdev.doclinks._qual_mod': ('api/doclinks.html#_qual_mod', 'nbdev/doclinks.py'),
@@ -164,6 +165,7 @@
                                'nbdev.process._is_direc': ('api/process.html#_is_direc', 'nbdev/process.py'),
                                'nbdev.process._mk_procs': ('api/process.html#_mk_procs', 'nbdev/process.py'),
                                'nbdev.process._norm_quarto': ('api/process.html#_norm_quarto', 'nbdev/process.py'),
+                               'nbdev.process._partition_cell': ('api/process.html#_partition_cell', 'nbdev/process.py'),
                                'nbdev.process._quarto_re': ('api/process.html#_quarto_re', 'nbdev/process.py'),
                                'nbdev.process.extract_directives': ('api/process.html#extract_directives', 'nbdev/process.py'),
                                'nbdev.process.first_code_ln': ('api/process.html#first_code_ln', 'nbdev/process.py'),
@@ -317,11 +319,10 @@
                                'nbdev.showdoc.doc': ('api/showdoc.html#doc', 'nbdev/showdoc.py'),
                                'nbdev.showdoc.show_doc': ('api/showdoc.html#show_doc', 'nbdev/showdoc.py'),
                                'nbdev.showdoc.showdoc_nm': ('api/showdoc.html#showdoc_nm', 'nbdev/showdoc.py')},
-            'nbdev.sync': { 'nbdev.sync._get_call': ('api/sync.html#_get_call', 'nbdev/sync.py'),
-                            'nbdev.sync._mod_files': ('api/sync.html#_mod_files', 'nbdev/sync.py'),
-                            'nbdev.sync._script2notebook': ('api/sync.html#_script2notebook', 'nbdev/sync.py'),
+            'nbdev.sync': { 'nbdev.sync._mod_files': ('api/sync.html#_mod_files', 'nbdev/sync.py'),
                             'nbdev.sync._to_absolute': ('api/sync.html#_to_absolute', 'nbdev/sync.py'),
-                            'nbdev.sync._update_lib': ('api/sync.html#_update_lib', 'nbdev/sync.py'),
+                            'nbdev.sync._update_mod': ('api/sync.html#_update_mod', 'nbdev/sync.py'),
+                            'nbdev.sync._update_nb': ('api/sync.html#_update_nb', 'nbdev/sync.py'),
                             'nbdev.sync.absolute_import': ('api/sync.html#absolute_import', 'nbdev/sync.py'),
                             'nbdev.sync.nbdev_update': ('api/sync.html#nbdev_update', 'nbdev/sync.py')},
             'nbdev.test': { 'nbdev.test._keep_file': ('api/test.html#_keep_file', 'nbdev/test.py'),

nbdev/doclinks.py

@@ -43,35 +43,43 @@ def patch_name(o):
     return _sym_nm(a,o)
 
 # %% ../nbs/api/doclinks.ipynb 9
-def _nbpath2html(p): return p.with_name(re.sub(r'\d+[a-zA-Z0-9]*_', '', p.name.lower())).with_suffix('.html')
+def _iter_py_cells(p):
+    "Yield cells from an exported Python file."
+    p = Path(p)
+    cells = p.read_text().split("\n# %% ")
+    for cell in cells[1:]:
+        top,code = cell.split('\n', 1)
+        nb,idx = top.split()
+        nb_path = None if nb=='auto' else (p.parent/nb).resolve()  # NB paths are stored relative to .py file
+        if code.endswith('\n'): code=code[:-1]
+        yield AttrDict(nb=nb, idx=int(idx), code=code, nb_path=nb_path, py_path=p.resolve())
 
 # %% ../nbs/api/doclinks.ipynb 11
-def _get_modidx(pyfile, code_root, nbs_path):
+def _nbpath2html(p): return p.with_name(re.sub(r'\d+[a-zA-Z0-9]*_', '', p.name.lower())).with_suffix('.html')
+
+# %% ../nbs/api/doclinks.ipynb 13
+def _get_modidx(py_path, code_root, nbs_path):
     "Get module symbol index for a Python source file"
     cfg = get_config()
-    rel_name = pyfile.resolve().relative_to(code_root).as_posix()
-    mod_name = '.'.join(rel_name.rpartition('.')[0].split('/'))  # module name created by pyfile
-    cells = Path(pyfile).read_text().split("\n# %% ")
+    rel_name = py_path.resolve().relative_to(code_root).as_posix()
+    mod_name = '.'.join(rel_name.rpartition('.')[0].split('/'))  # module name created by py_path
 
     _def_types = ast.FunctionDef,ast.AsyncFunctionDef,ast.ClassDef
     d = {}
-    for cell in cells[1:]:  # First cell is autogenerated header
-        top,*rest = cell.splitlines() # First line is cell header
-        nb = top.split()[0]
-        if nb != 'auto':
-            nbpath = (pyfile.parent/nb).resolve()  # NB paths are stored relative to .py file
-            loc = _nbpath2html(nbpath.relative_to(nbs_path))
-
-            def _stor(nm):
-                for n in L(nm): d[f'{mod_name}.{n}'] = f'{loc.as_posix()}#{n.lower()}',rel_name
-            for tree in ast.parse('\n'.join(rest)).body:
-                if isinstance(tree, _def_types): _stor(patch_name(tree))
-                if isinstance(tree, ast.ClassDef):
-                    for t2 in tree.body:
-                        if isinstance(t2, _def_types): _stor(f'{tree.name}.{t2.name}')
+    for cell in _iter_py_cells(py_path):
+        if cell.nb == 'auto': continue
+        loc = _nbpath2html(cell.nb_path.relative_to(nbs_path))
+
+        def _stor(nm):
+            for n in L(nm): d[f'{mod_name}.{n}'] = f'{loc.as_posix()}#{n.lower()}',rel_name
+        for tree in ast.parse(cell.code).body:
+            if isinstance(tree, _def_types): _stor(patch_name(tree))
+            if isinstance(tree, ast.ClassDef):
+                for t2 in tree.body:
+                    if isinstance(t2, _def_types): _stor(f'{tree.name}.{t2.name}')
     return {mod_name: d}
 
-# %% ../nbs/api/doclinks.ipynb 12
+# %% ../nbs/api/doclinks.ipynb 15
 def _build_modidx(dest=None, nbs_path=None, skip_exists=False):
     "Create _modidx.py"
     if dest is None: dest = get_config().lib_path
@@ -89,7 +97,7 @@ def _build_modidx(dest=None, nbs_path=None, skip_exists=False):
         res['syms'].update(_get_modidx((dest.parent/file).resolve(), code_root, nbs_path=nbs_path))
     idxfile.write_text("# Autogenerated by nbdev\n\nd = "+pformat(res, width=140, indent=2, compact=True))
 
-# %% ../nbs/api/doclinks.ipynb 17
+# %% ../nbs/api/doclinks.ipynb 20
 @delegates(globtastic)
 def nbglob(path=None, skip_folder_re = '^[_.]', file_glob='*.ipynb', skip_file_re='^[_.]', key='nbs_path', as_path=False, **kwargs):
     "Find all files in a directory matching an extension given a config key."
@@ -99,7 +107,7 @@ def nbglob(path=None, skip_folder_re = '^[_.]', file_glob='*.ipynb', skip_file_r
                      skip_file_re=skip_file_re, recursive=recursive, **kwargs)
     return res.map(Path) if as_path else res
 
-# %% ../nbs/api/doclinks.ipynb 18
+# %% ../nbs/api/doclinks.ipynb 21
 def nbglob_cli(
     path:str=None, # Path to notebooks
     symlinks:bool=False, # Follow symlinks?
@@ -113,7 +121,7 @@ def nbglob_cli(
     return nbglob(path, symlinks=symlinks, file_glob=file_glob, file_re=file_re, folder_re=folder_re,
                   skip_file_glob=skip_file_glob, skip_file_re=skip_file_re, skip_folder_re=skip_folder_re)
 
-# %% ../nbs/api/doclinks.ipynb 19
+# %% ../nbs/api/doclinks.ipynb 22
 @call_parse
 @delegates(nbglob_cli)
 def nbdev_export(
@@ -126,11 +134,11 @@ def nbdev_export(
     add_init(get_config().lib_path)
     _build_modidx()
 
-# %% ../nbs/api/doclinks.ipynb 21
+# %% ../nbs/api/doclinks.ipynb 24
 import importlib,ast
 from functools import lru_cache
 
-# %% ../nbs/api/doclinks.ipynb 22
+# %% ../nbs/api/doclinks.ipynb 25
 def _find_mod(mod):
     mp,_,mr = mod.partition('/')
     spec = importlib.util.find_spec(mp)
@@ -153,7 +161,7 @@ def _get_exps(mod):
 
 def _lineno(sym, fname): return _get_exps(fname).get(sym, None) if fname else None
 
-# %% ../nbs/api/doclinks.ipynb 24
+# %% ../nbs/api/doclinks.ipynb 27
 def _qual_sym(s, settings):
     if not isinstance(s,tuple): return s
     nb,py = s
@@ -168,10 +176,10 @@ def _qual_syms(entries):
     if 'doc_host' not in settings: return entries
     return {'syms': {mod:_qual_mod(d, settings) for mod,d in entries['syms'].items()}, 'settings':settings}
 
-# %% ../nbs/api/doclinks.ipynb 25
+# %% ../nbs/api/doclinks.ipynb 28
 _re_backticks = re.compile(r'`([^`\s]+)`')
 
-# %% ../nbs/api/doclinks.ipynb 26
+# %% ../nbs/api/doclinks.ipynb 29
 @lru_cache(None)
 class NbdevLookup:
     "Mapping from symbol names to docs and source URLs"

nbdev/maker.py

@@ -149,7 +149,7 @@ def update_import(source, tree, libname, f=relative_import):
         nmod = f(imp.module, libname, imp.level)
         lin = imp.lineno-1
         sec = src[lin][imp.col_offset:imp.end_col_offset]
-        newsec = re.sub(f"(from +){'.'*imp.level}{imp.module}", fr"\1{nmod}", sec)
+        newsec = re.sub(f"(from +){'.'*imp.level}{imp.module or ''}", fr"\1{nmod}", sec)
         src[lin] = src[lin].replace(sec,newsec)
     return src
 

nbdev/process.py

@@ -54,34 +54,38 @@ def first_code_ln(code_list, re_pattern=None, lang='python'):
     return first(i for i,o in enumerate(code_list) if o.strip() != '' and not re.match(re_pattern, o) and not _cell_mgc.match(o))
 
 # %% ../nbs/api/process.ipynb 17
+def _partition_cell(cell, lang):
+    if not cell.source: return [],[]
+    lines = cell.source.splitlines(True)
+    first_code = first_code_ln(lines, lang=lang)
+    return lines[:first_code],lines[first_code:]
+
+# %% ../nbs/api/process.ipynb 18
 def extract_directives(cell, remove=True, lang='python'):
     "Take leading comment directives from lines of code in `ss`, remove `#|`, and split"
-    if cell.source:
-        ss = cell.source.splitlines(True)
-        first_code = first_code_ln(ss, lang=lang)
-        if not ss or first_code==0: return {}
-        pre = ss[:first_code]
-        if remove:
-            # Leave Quarto directives and cell magic in place for later processing
-            cell['source'] = ''.join([_norm_quarto(o, lang) for o in pre if _quarto_re(lang).match(o) or _cell_mgc.match(o)] + ss[first_code:])
-        return dict(L(_directive(s, lang) for s in pre).filter())
-
-# %% ../nbs/api/process.ipynb 21
+    dirs,code = _partition_cell(cell, lang)
+    if not dirs: return {}
+    if remove:
+        # Leave Quarto directives and cell magic in place for later processing
+        cell['source'] = ''.join([_norm_quarto(o, lang) for o in dirs if _quarto_re(lang).match(o) or _cell_mgc.match(o)] + code)
+    return dict(L(_directive(s, lang) for s in dirs).filter())
+
+# %% ../nbs/api/process.ipynb 22
 def opt_set(var, newval):
     "newval if newval else var"
     return newval if newval else var
 
-# %% ../nbs/api/process.ipynb 22
+# %% ../nbs/api/process.ipynb 23
 def instantiate(x, **kwargs):
     "Instantiate `x` if it's a type"
     return x(**kwargs) if isinstance(x,type) else x
 
 def _mk_procs(procs, nb): return L(procs).map(instantiate, nb=nb)
 
-# %% ../nbs/api/process.ipynb 23
+# %% ../nbs/api/process.ipynb 24
 def _is_direc(f): return getattr(f, '__name__', '-')[-1]=='_'
 
-# %% ../nbs/api/process.ipynb 24
+# %% ../nbs/api/process.ipynb 25
 class NBProcessor:
     "Process cells and nbdev comments in a notebook"
     def __init__(self, path=None, procs=None, nb=None, debug=False, rm_directives=True, process=False):
@@ -121,7 +125,7 @@ def process(self):
         "Process all cells with all processors"
         for proc in self.procs: self._proc(proc)
 
-# %% ../nbs/api/process.ipynb 34
+# %% ../nbs/api/process.ipynb 35
 class Processor:
     "Base class for processors"
     def __init__(self, nb): self.nb = nb

nbdev/sync.py

@@ -8,7 +8,9 @@
 from .config import *
 from .maker import *
 from .process import *
+from .process import _partition_cell
 from .export import *
+from .doclinks import _iter_py_cells
 
 from execnb.nbio import *
 from fastcore.script import *
@@ -26,51 +28,46 @@ def absolute_import(name, fname, level):
     return '.'.join(mods[:len(mods)-level+1]) + f".{name}"
 
 # %% ../nbs/api/sync.ipynb 7
+@functools.lru_cache(maxsize=None)
+def _mod_files():
+    midx = import_module(f'{get_config().lib_name}._modidx')
+    return L(files for mod in midx.d['syms'].values() for _,files in mod.values()).unique()
+
+# %% ../nbs/api/sync.ipynb 8
 _re_import = re.compile("from\s+\S+\s+import\s+\S")
 
-# %% ../nbs/api/sync.ipynb 9
-def _to_absolute(code, lib_name):
+# %% ../nbs/api/sync.ipynb 10
+def _to_absolute(code, py_path, lib_dir):
     if not _re_import.search(code): return code
-    res = update_import(code, ast.parse(code).body, lib_name, absolute_import)
+    res = update_import(code, ast.parse(code).body, str(py_path.relative_to(lib_dir).parent), absolute_import)
     return ''.join(res) if res else code
 
-def _update_lib(nbname, nb_locs, lib_name=None):
-    if lib_name is None: lib_name = get_config().lib_name
-    absnm = get_config().path('lib_path')/nbname
-    nbp = NBProcessor(absnm, ExportModuleProc(), rm_directives=False)
+# %% ../nbs/api/sync.ipynb 11
+def _update_nb(nb_path, cells, lib_dir):
+    "Update notebook `nb_path` with contents from `cells`"
+    nbp = NBProcessor(nb_path, ExportModuleProc(), rm_directives=False)
     nbp.process()
-    nb = nbp.nb
-
-    for name,idx,code in nb_locs:
-        assert name==nbname
-        cell = nb.cells[int(idx)]
-        directives = ''.join(cell.source.splitlines(True)[:len(cell.directives_)])
-        cell.source = directives + _to_absolute(code, lib_name)
-    write_nb(nb, absnm)
-
-# %% ../nbs/api/sync.ipynb 10
-@functools.lru_cache(maxsize=None)
-def _mod_files():
-    midx = import_module(f'{get_config().lib_name}._modidx')
-    return L(files for mod in midx.d['syms'].values() for _,files in mod.values()).unique()
+    for cell in cells:
+        assert cell.nb_path == nb_path
+        nbcell = nbp.nb.cells[cell.idx]
+        dirs,_ = _partition_cell(nbcell, 'python')
+        nbcell.source = ''.join(dirs) + _to_absolute(cell.code, cell.py_path, lib_dir)
+    write_nb(nbp.nb, nb_path)
 
 # %% ../nbs/api/sync.ipynb 12
-def _get_call(s):
-    top,*rest = s.splitlines()
-    return (*top.split(),'\n'.join(rest))
-
-def _script2notebook(fname:str):
-    code_cells = Path(fname).read_text().split("\n# %% ")[1:]
-    locs = L(_get_call(s) for s in code_cells if not s.startswith('auto '))
-    for nbname,nb_locs in groupby(locs, 0).items(): _update_lib(nbname, nb_locs)
+def _update_mod(py_path, lib_dir):
+    "Propagate changes from cells in module `py_path` to corresponding notebooks"
+    py_cells = L(_iter_py_cells(py_path)).filter(lambda o: o.nb != 'auto')
+    for nb_path,cells in groupby(py_cells, 'nb_path').items(): _update_nb(nb_path, cells, lib_dir)
 
+# %% ../nbs/api/sync.ipynb 14
 @call_parse
 def nbdev_update(fname:str=None): # A Python file name to update
     "Propagate change in modules matching `fname` to notebooks that created them"
     if fname and fname.endswith('.ipynb'): raise ValueError("`nbdev_update` operates on .py files.  If you wish to convert notebooks instead, see `nbdev_export`.")
     if os.environ.get('IN_TEST',0): return
-    fname = Path(fname or get_config().path('lib_path'))
-    lib_dir = get_config().path("lib_path").parent
+    cfg = get_config()
+    fname = Path(fname or cfg.lib_path)
+    lib_dir = cfg.lib_path.parent
     files = globtastic(fname, file_glob='*.py').filter(lambda x: str(Path(x).absolute().relative_to(lib_dir) in _mod_files()))
-    files.map(_script2notebook)
-
+    files.map(_update_mod, lib_dir=lib_dir)