From c93ea7202401343f49e019b4ddfb6dcc0d3016db Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Tue, 27 May 2025 12:15:42 -0400
Subject: [PATCH 01/31] Add .qmd read functionality to processor
---
nbdev/_modidx.py | 25 ++-
nbdev/process.py | 113 ++++++++++++-
nbdev/read.py | 327 ++++++++++++++++++++++++++++++++++++++
nbs/api/03_process.ipynb | 220 +++++++++++++++++++++++--
tests/01_everything.ipynb | 156 +++++++++++++++---
tests/01_everything.qmd | 183 +++++++++++++++++++++
tests/minimal.qmd | 7 +
7 files changed, 990 insertions(+), 41 deletions(-)
create mode 100644 nbdev/read.py
create mode 100644 tests/01_everything.qmd
create mode 100644 tests/minimal.qmd
diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py
index 650ce4a56..caf820192 100644
--- a/nbdev/_modidx.py
+++ b/nbdev/_modidx.py
@@ -178,11 +178,13 @@
'nbdev.process._norm_quarto': ('api/process.html#_norm_quarto', 'nbdev/process.py'),
'nbdev.process._partition_cell': ('api/process.html#_partition_cell', 'nbdev/process.py'),
'nbdev.process._quarto_re': ('api/process.html#_quarto_re', 'nbdev/process.py'),
+ 'nbdev.process._read_nb_or_qmd': ('api/process.html#_read_nb_or_qmd', 'nbdev/process.py'),
'nbdev.process.extract_directives': ('api/process.html#extract_directives', 'nbdev/process.py'),
'nbdev.process.first_code_ln': ('api/process.html#first_code_ln', 'nbdev/process.py'),
'nbdev.process.instantiate': ('api/process.html#instantiate', 'nbdev/process.py'),
'nbdev.process.nb_lang': ('api/process.html#nb_lang', 'nbdev/process.py'),
- 'nbdev.process.opt_set': ('api/process.html#opt_set', 'nbdev/process.py')},
+ 'nbdev.process.opt_set': ('api/process.html#opt_set', 'nbdev/process.py'),
+ 'nbdev.process.read_qmd': ('api/process.html#read_qmd', 'nbdev/process.py')},
'nbdev.processors': { 'nbdev.processors.FilterDefaults': ('api/processors.html#filterdefaults', 'nbdev/processors.py'),
'nbdev.processors.FilterDefaults.__call__': ( 'api/processors.html#filterdefaults.__call__',
'nbdev/processors.py'),
@@ -281,6 +283,27 @@
'nbdev.quarto.nbdev_sidebar': ('api/quarto.html#nbdev_sidebar', 'nbdev/quarto.py'),
'nbdev.quarto.prepare': ('api/quarto.html#prepare', 'nbdev/quarto.py'),
'nbdev.quarto.refresh_quarto_yml': ('api/quarto.html#refresh_quarto_yml', 'nbdev/quarto.py')},
+ 'nbdev.read': { 'nbdev.read._apply_defaults': ('api/config.html#_apply_defaults', 'nbdev/read.py'),
+ 'nbdev.read._basic_export_nb': ('api/config.html#_basic_export_nb', 'nbdev/read.py'),
+ 'nbdev.read._cfg2txt': ('api/config.html#_cfg2txt', 'nbdev/read.py'),
+ 'nbdev.read._fetch_from_git': ('api/config.html#_fetch_from_git', 'nbdev/read.py'),
+ 'nbdev.read._get_info': ('api/config.html#_get_info', 'nbdev/read.py'),
+ 'nbdev.read._git_repo': ('api/config.html#_git_repo', 'nbdev/read.py'),
+ 'nbdev.read._has_py': ('api/config.html#_has_py', 'nbdev/read.py'),
+ 'nbdev.read._nbdev_config_file': ('api/config.html#_nbdev_config_file', 'nbdev/read.py'),
+ 'nbdev.read._prompt_user': ('api/config.html#_prompt_user', 'nbdev/read.py'),
+ 'nbdev.read._type': ('api/config.html#_type', 'nbdev/read.py'),
+ 'nbdev.read._xdg_config_paths': ('api/config.html#_xdg_config_paths', 'nbdev/read.py'),
+ 'nbdev.read.add_init': ('api/config.html#add_init', 'nbdev/read.py'),
+ 'nbdev.read.config_key': ('api/config.html#config_key', 'nbdev/read.py'),
+ 'nbdev.read.create_output': ('api/config.html#create_output', 'nbdev/read.py'),
+ 'nbdev.read.get_config': ('api/config.html#get_config', 'nbdev/read.py'),
+ 'nbdev.read.is_nbdev': ('api/config.html#is_nbdev', 'nbdev/read.py'),
+ 'nbdev.read.nbdev_create_config': ('api/config.html#nbdev_create_config', 'nbdev/read.py'),
+ 'nbdev.read.show_src': ('api/config.html#show_src', 'nbdev/read.py'),
+ 'nbdev.read.update_proj': ('api/config.html#update_proj', 'nbdev/read.py'),
+ 'nbdev.read.update_version': ('api/config.html#update_version', 'nbdev/read.py'),
+ 'nbdev.read.write_cells': ('api/config.html#write_cells', 'nbdev/read.py')},
'nbdev.release': { 'nbdev.release.Release': ('api/release.html#release', 'nbdev/release.py'),
'nbdev.release.Release.__init__': ('api/release.html#release.__init__', 'nbdev/release.py'),
'nbdev.release.Release._issue_groups': ('api/release.html#release._issue_groups', 'nbdev/release.py'),
diff --git a/nbdev/process.py b/nbdev/process.py
index 9e8e4f5f7..205f71b58 100644
--- a/nbdev/process.py
+++ b/nbdev/process.py
@@ -3,7 +3,8 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/03_process.ipynb.
# %% auto 0
-__all__ = ['langs', 'nb_lang', 'first_code_ln', 'extract_directives', 'opt_set', 'instantiate', 'NBProcessor', 'Processor']
+__all__ = ['langs', 'read_qmd', 'nb_lang', 'first_code_ln', 'extract_directives', 'opt_set', 'instantiate', 'NBProcessor',
+ 'Processor']
# %% ../nbs/api/03_process.ipynb
from .config import *
@@ -16,6 +17,109 @@
from collections import defaultdict
+# %% ../nbs/api/03_process.ipynb
+import re
+import json
+from pathlib import Path
+from fastcore.utils import AttrDict
+
+# Assuming AttrDict is available from fastcore.utils or similar
+# If not, a regular dictionary can be used for the structure
+# from fastcore.utils import AttrDict
+
+def read_qmd(path):
+ """Return notebook-like structure from a .qmd file"""
+ path = Path(path)
+ content = path.read_text(encoding='utf-8')
+
+ cells = []
+ current_cell_source = []
+ current_cell_type = 'markdown' # Start assuming markdown
+
+ # Regex to find fenced code blocks with language identifier
+ code_block_start_re = re.compile(r'^```{\s*(\w+)\s*}')
+ code_block_end_re = re.compile(r'^```\s*$')
+
+ in_code_block = False
+
+ # Iterate through lines to identify markdown and code cells
+ for line in content.splitlines(keepends=True):
+ code_match = code_block_start_re.match(line)
+ code_end_match = code_block_end_re.match(line)
+
+ if code_match:
+ # If we were in a markdown cell, save it first
+ if current_cell_source and current_cell_type == 'markdown':
+ cells.append({
+ 'cell_type': 'markdown',
+ 'metadata': {},
+ 'source': ''.join(current_cell_source)
+ })
+ current_cell_source = []
+
+ # Start a new code cell
+ current_cell_type = 'code'
+ in_code_block = True
+ # We skip the ````{language}` line itself in the cell source
+
+ elif code_end_match and in_code_block:
+ # End of a code block, save the code cell
+ if current_cell_source:
+ # Remove the last line if it's just the closing fence ```
+ # (splitlines(keepends=True) includes the newline)
+ if current_cell_source[-1].strip() == '```':
+ current_cell_source.pop()
+ cells.append({
+ 'cell_type': 'code',
+ 'execution_count': None, # .qmd files don't store execution count
+ 'metadata': {}, # Metadata might be inferred later or added default
+ 'outputs': [], # .qmd files don't store outputs
+ 'source': ''.join(current_cell_source)
+ })
+ current_cell_source = []
+ current_cell_type = 'markdown' # Next content is markdown
+ in_code_block = False
+
+ else:
+ # Add line to the current cell source
+ current_cell_source.append(line)
+
+ # Add any remaining markdown content as a cell
+ if current_cell_source and current_cell_type == 'markdown':
+ cells.append({
+ 'cell_type': 'markdown',
+ 'metadata': {},
+ 'source': ''.join(current_cell_source)
+ })
+
+ # Construct the final notebook-like dictionary
+ # Use default values for nbformat and metadata that mimic ipynb
+ nb_obj = {
+ 'cells': cells,
+ 'metadata': {
+ 'kernelspec': {
+ 'display_name': 'Python 3', # Default, could try to infer from ```{lang}
+ 'language': 'python', # Default
+ 'name': 'python3' # Default
+ },
+ 'language_info': { # Basic default language info
+ 'name': 'python'
+ }
+ },
+ 'nbformat': 4,
+ 'nbformat_minor': 5, # Use a recent minor version
+ 'path_': str(path)
+ }
+
+ # If AttrDict is desired, convert here:
+ return dict2nb(nb_obj)
+ # return nb_obj
+
+# Example Usage (assuming minimal.qmd is in a 'tests' directory relative to execution)
+# qmd_path = 'tests/minimal.qmd'
+# nb_like_object = read_qmd(qmd_path)
+# print(json.dumps(nb_like_object, indent=2)) # Print to see the structure
+
# %% ../nbs/api/03_process.ipynb
# from https://github.com/quarto-dev/quarto-cli/blob/main/src/resources/jupyter/notebook.py
langs = defaultdict(
@@ -88,11 +192,16 @@ def _mk_procs(procs, nb): return L(procs).map(instantiate, nb=nb)
# %% ../nbs/api/03_process.ipynb
def _is_direc(f): return getattr(f, '__name__', '-')[-1]=='_'
+# %% ../nbs/api/03_process.ipynb
+def _read_nb_or_qmd(path):
+ if Path(path).suffix == '.qmd': return read_qmd(path)
+ return read_nb(path)
+
# %% ../nbs/api/03_process.ipynb
class NBProcessor:
"Process cells and nbdev comments in a notebook"
def __init__(self, path=None, procs=None, nb=None, debug=False, rm_directives=True, process=False):
- self.nb = read_nb(path) if nb is None else nb
+ self.nb = _read_nb_or_qmd(path) if nb is None else nb
self.lang = nb_lang(self.nb)
for cell in self.nb.cells: cell.directives_ = extract_directives(cell, remove=rm_directives, lang=self.lang)
self.procs = _mk_procs(procs, nb=self.nb)
diff --git a/nbdev/read.py b/nbdev/read.py
new file mode 100644
index 000000000..631a54d28
--- /dev/null
+++ b/nbdev/read.py
@@ -0,0 +1,327 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/01_config.ipynb.
+
+# %% auto 0
+__all__ = ['pyproj_tmpl', 'nbdev_create_config', 'get_config', 'config_key', 'is_nbdev', 'create_output', 'show_src',
+ 'update_version', 'update_proj', 'add_init', 'write_cells']
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+from datetime import datetime
+from fastcore.docments import *
+from fastcore.utils import *
+from fastcore.meta import *
+from fastcore.script import *
+from fastcore.style import *
+from fastcore.xdg import *
+
+import ast
+from IPython.display import Markdown
+from execnb.nbio import read_nb,NbCell
+from urllib.error import HTTPError
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+_nbdev_home_dir = 'nbdev' # sub-directory of xdg base dir
+_nbdev_cfg_name = 'settings.ini'
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+def _git_repo():
+ try: return repo_details(run('git config --get remote.origin.url'))[1]
+ except OSError: return
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+
+# When adding a named default to the list below, be sure that that name
+# is also added to one of the sections in `_nbdev_cfg_sections` as well,
+# or it won't get written by `nbdev_create_config`:
+def _apply_defaults(
+ cfg,
+ lib_name='%(repo)s', # Package name
+ git_url='https://github.com/%(user)s/%(repo)s', # Repo URL
+ custom_sidebar:bool_arg=False, # Use a custom sidebar.yml?
+ nbs_path:Path='nbs', # Path to notebooks
+ lib_path:Path=None, # Path to package root (default: `repo` with `-` replaced by `_`)
+ doc_path:Path='_docs', # Path to rendered docs
+ tst_flags='notest', # Test flags
+ version='0.0.1', # Version of this release
+ doc_host='https://%(user)s.github.io', # Hostname for docs
+ doc_baseurl='/%(repo)s', # Base URL for docs
+ keywords='nbdev jupyter notebook python', # Package keywords
+ license='apache2', # License for the package
+ copyright:str=None, # Copyright for the package, defaults to '`current_year` onwards, `author`'
+ status='3', # Development status PyPI classifier
+ min_python='3.9', # Minimum Python version PyPI classifier
+ audience='Developers', # Intended audience PyPI classifier
+ language='English', # Language PyPI classifier
+ recursive:bool_arg=True, # Include subfolders in notebook globs?
+ black_formatting:bool_arg=False, # Format libraries with black?
+ readme_nb='index.ipynb', # Notebook to export as repo readme
+ title='%(lib_name)s', # Quarto website title
+ allowed_metadata_keys='', # Preserve the list of keys in the main notebook metadata
+ allowed_cell_metadata_keys='', # Preserve the list of keys in cell level metadata
+ jupyter_hooks:bool_arg=False, # Run Jupyter hooks?
+ clean_ids:bool_arg=True, # Remove ids from plaintext reprs?
+ clear_all:bool_arg=False, # Remove all cell metadata and cell outputs?
+ cell_number:bool_arg=True, # Add cell number to the exported file
+ put_version_in_init:bool_arg=True, # Add the version to the main __init__.py in nbdev_export
+ update_pyproject:bool_arg=True, # Create/update pyproject.toml with correct project name
+ skip_procs:str='', # A comma-separated list of processors that you want to skip
+):
+ "Apply default settings where missing in `cfg`."
+ if getattr(cfg,'repo',None) is None:
+ cfg.repo = _git_repo()
+ if cfg.repo is None:
+ _parent = Path.cwd()
+ cfg.repo = _parent.parent.name if _parent.name=='nbs' else _parent.name
+ if lib_path is None: lib_path = cfg.repo.replace('-', '_')
+ if copyright is None: copyright = f"{datetime.now().year} onwards, %(author)s"
+ for k,v in locals().items():
+ if k.startswith('_') or k == 'cfg' or cfg.get(k) is not None: continue
+ cfg[k] = v
+ return cfg
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+def _get_info(owner, repo, default_branch='main', default_kw='nbdev'):
+ from ghapi.all import GhApi
+ api = GhApi(owner=owner, repo=repo, token=os.getenv('GITHUB_TOKEN'))
+
+ try: r = api.repos.get()
+ except HTTPError:
+ msg= [f"""Could not access repo: {owner}/{repo} to find your default branch - `{default_branch}` assumed.
+Edit `settings.ini` if this is incorrect.
+In the future, you can allow nbdev to see private repos by setting the environment variable GITHUB_TOKEN as described here:
+https://nbdev.fast.ai/api/release.html#setup"""]
+ print(''.join(msg))
+ return default_branch,default_kw,''
+
+ return r.default_branch, default_kw if not getattr(r, 'topics', []) else ' '.join(r.topics), r.description
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+def _fetch_from_git(raise_err=False):
+ "Get information for settings.ini from the user."
+ res={}
+ try:
+ url = run('git config --get remote.origin.url')
+ res['user'],res['repo'] = repo_details(url)
+ res['branch'],res['keywords'],desc = _get_info(owner=res['user'], repo=res['repo'])
+ if desc: res['description'] = desc
+ res['author'] = run('git config --get user.name').strip() # below two lines attempt to pull from global user config
+ res['author_email'] = run('git config --get user.email').strip()
+ except OSError as e:
+ if raise_err: raise(e)
+ else: res['lib_name'] = res['repo'].replace('-','_')
+ return res
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+def _prompt_user(cfg, inferred):
+ "Let user input values not in `cfg` or `inferred`."
+ res = cfg.copy()
+ for k,v in cfg.items():
+ inf = inferred.get(k,None)
+ msg = S.light_blue(k) + ' = '
+ if v is None:
+ if inf is None: res[k] = input(f'# Please enter a value for {k}\n'+msg)
+ else:
+ res[k] = inf
+ print(msg+res[k]+' # Automatically inferred from git')
+ return res
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+def _cfg2txt(cfg, head, sections, tail=''):
+ "Render `cfg` with commented sections."
+ nm = cfg.d.name
+ res = f'[{nm}]\n'+head
+ for t,ks in sections.items():
+ res += f'### {t} ###\n'
+ for k in ks.split(): res += f"{k} = {cfg._cfg.get(nm, k, raw=True)}\n" # TODO: add `raw` to `Config.get`
+ res += '\n'
+ res += tail
+ return res.strip()
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+_nbdev_cfg_head = '''# All sections below are required unless otherwise specified.
+# See https://github.com/AnswerDotAI/nbdev/blob/main/settings.ini for examples.
+
+'''
+_nbdev_cfg_sections = {'Python library': 'repo lib_name version min_python license black_formatting',
+ 'nbdev': 'doc_path lib_path nbs_path recursive tst_flags put_version_in_init update_pyproject',
+ 'Docs': 'branch custom_sidebar doc_host doc_baseurl git_url title',
+ 'PyPI': 'audience author author_email copyright description keywords language status user'}
+_nbdev_cfg_tail = '''### Optional ###
+# requirements = fastcore pandas
+# dev_requirements =
+# console_scripts =
+# conda_user =
+# package_data =
+'''
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+@call_parse
+@delegates(_apply_defaults, but='cfg')
+def nbdev_create_config(
+ repo:str=None, # Repo name
+ branch:str=None, # Repo default branch
+ user:str=None, # Repo username
+ author:str=None, # Package author's name
+ author_email:str=None, # Package author's email address
+ description:str=None, # Short summary of the package
+ path:str='.', # Path to create config file
+ cfg_name:str=_nbdev_cfg_name, # Name of config file to create
+ **kwargs
+):
+ "Create a config file."
+ req = {k:v for k,v in locals().items() if k not in ('path','cfg_name','kwargs')}
+ inf = _fetch_from_git()
+ d = _prompt_user(req, inf)
+ cfg = Config(path, cfg_name, d, save=False)
+ if cfg.config_file.exists(): warn(f'Config file already exists: {cfg.config_file} and will be used as a base')
+ cfg = _apply_defaults(cfg, **kwargs)
+ txt = _cfg2txt(cfg, _nbdev_cfg_head, _nbdev_cfg_sections, _nbdev_cfg_tail)
+ cfg.config_file.write_text(txt)
+ cfg_fn = Path(path)/cfg_name
+ print(f'{cfg_fn} created.')
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+def _nbdev_config_file(cfg_name=_nbdev_cfg_name, path=None):
+ cfg_path = Path.cwd() if path is None else Path(path)
+ return getattr(Config.find(cfg_name), 'config_file', cfg_path/cfg_name)
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+def _xdg_config_paths(cfg_name=_nbdev_cfg_name):
+ xdg_config_paths = reversed([xdg_config_home()]+xdg_config_dirs())
+ return [o/_nbdev_home_dir/cfg_name for o in xdg_config_paths]
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+def _type(t): return bool if t==bool_arg else t
+_types = {k:_type(v['anno']) for k,v in docments(_apply_defaults,full=True,returns=False).items() if k != 'cfg'}
+
+@functools.lru_cache(maxsize=None)
+def get_config(cfg_name=_nbdev_cfg_name, path=None):
+ "Return nbdev config."
+ cfg_file = _nbdev_config_file(cfg_name, path)
+ extra_files = _xdg_config_paths(cfg_name)
+ cfg = Config(cfg_file.parent, cfg_file.name, extra_files=extra_files, types=_types)
+ return _apply_defaults(cfg)
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+def config_key(c, default=None, path=True, missing_ok=None):
+ "Deprecated: use `get_config().get` or `get_config().path` instead."
+ warn("`config_key` is deprecated. Use `get_config().get` or `get_config().path` instead.", DeprecationWarning)
+ return get_config().path(c, default) if path else get_config().get(c, default)
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+def is_nbdev(): return _nbdev_config_file().exists()
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+def create_output(txt, mime):
+ "Add a cell output containing `txt` of the `mime` text MIME sub-type"
+ return [{"data": { f"text/{mime}": str(txt).splitlines(True) },
+ "execution_count": 1, "metadata": {}, "output_type": "execute_result"}]
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+def show_src(src, lang='python'): return Markdown(f'```{lang}\n{src}\n```')
+
+# %% ../nbs/api/01_config.ipynb
+#| export
+pyproj_tmpl = """[build-system]
+requires = ["setuptools>=64.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "FILL_IN"
+requires-python="FILL_IN"
+dynamic = [ "keywords", "description", "version", "dependencies", "optional-dependencies", "readme", "license", "authors", "classifiers", "entry-points", "scripts", "urls"]
+
+[tool.uv]
+cache-keys = [{ file = "pyproject.toml" }, { file = "settings.ini" }, { file = "setup.py" }]
+"""
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+_re_version = re.compile(r'^__version__\s*=.*$', re.MULTILINE)
+_re_proj = re.compile(r'^name\s*=\s*".*$', re.MULTILINE)
+_re_reqpy = re.compile(r'^requires-python\s*=\s*".*$', re.MULTILINE)
+_init = '__init__.py'
+_pyproj = 'pyproject.toml'
+
+def update_version(path=None):
+ "Add or update `__version__` in the main `__init__.py` of the library."
+ path = Path(path or get_config().lib_path)
+ fname = path/_init
+ if not fname.exists(): fname.touch()
+ version = f'__version__ = "{get_config().version}"'
+ code = fname.read_text()
+ if _re_version.search(code) is None: code = version + "\n" + code
+ else: code = _re_version.sub(version, code)
+ fname.write_text(code)
+
+def _has_py(fs): return any(1 for f in fs if f.endswith('.py'))
+
+def update_proj(path):
+ "Create or update `pyproject.toml` in the project root."
+ fname = path/_pyproj
+ if not fname.exists(): fname.write_text(pyproj_tmpl)
+ txt = fname.read_text()
+ txt = _re_proj.sub(f'name="{get_config().lib_name}"', txt)
+ txt = _re_reqpy.sub(f'requires-python=">={get_config().min_python}"', txt)
+ fname.write_text(txt)
+
+def add_init(path=None):
+ "Add `__init__.py` in all subdirs of `path` containing python files if it's not there already."
+ # we add the lowest-level `__init__.py` files first, which ensures _has_py succeeds for parent modules
+ path = Path(path or get_config().lib_path)
+ path.mkdir(exist_ok=True)
+ if not (path/_init).exists(): (path/_init).touch()
+ for r,ds,fs in os.walk(path, topdown=False):
+ r = Path(r)
+ subds = (os.listdir(r/d) for d in ds)
+ if _has_py(fs) or any(filter(_has_py, subds)) and not (r/_init).exists(): (r/_init).touch()
+ if get_config().get('put_version_in_init', True): update_version(path)
+ if get_config().get('update_pyproject', True): update_proj(path.parent)
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+def write_cells(cells, hdr, file, offset=0, cell_number=True, solo_nb=False):
+ "Write `cells` to `file` along with header `hdr` starting at index `offset` (mainly for nbdev internal use)."
+ for cell in cells:
+ if cell.cell_type=='code' and cell.source.strip():
+ idx = f" {cell.idx_+offset}" if cell_number else ""
+ file.write(f'\n\n{hdr}{idx}\n{cell.source}') if not solo_nb else file.write(f'\n\n{cell.source}')
+
+# %% ../nbs/api/01_config.ipynb
+#|export
+def _basic_export_nb(fname, name, dest=None):
+ "Basic exporter to bootstrap nbdev."
+ if dest is None: dest = get_config().lib_path
+ add_init()
+ fname,dest = Path(fname),Path(dest)
+ nb = read_nb(fname)
+
+ # grab the source from all the cells that have an `export` comment
+ cells = L(cell for cell in nb.cells if re.match(r'#\s*\|export', cell.source))
+
+ # find all the exported functions, to create `__all__`:
+ trees = cells.map(NbCell.parsed_).concat()
+ funcs = trees.filter(risinstance((ast.FunctionDef,ast.ClassDef))).attrgot('name')
+ exp_funcs = [f for f in funcs if f[0]!='_']
+
+ # write out the file
+ with (dest/name).open('w',encoding="utf-8") as f:
+ f.write(f"# %% auto 0\n__all__ = {exp_funcs}")
+ write_cells(cells, f"# %% {fname.relpath(dest)}", f)
+ f.write('\n')
diff --git a/nbs/api/03_process.ipynb b/nbs/api/03_process.ipynb
index 6ae46e3dc..0afb5739e 100644
--- a/nbs/api/03_process.ipynb
+++ b/nbs/api/03_process.ipynb
@@ -69,7 +69,128 @@
"metadata": {},
"outputs": [],
"source": [
- "minimal = read_nb('../../tests/minimal.ipynb')"
+ "nb_minimal = read_nb('../../tests/minimal.ipynb')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "ca351f3d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "import re\n",
+ "import json\n",
+ "from pathlib import Path\n",
+ "from fastcore.utils import AttrDict\n",
+ "\n",
+ "# Assuming AttrDict is available from fastcore.utils or similar\n",
+ "# If not, a regular dictionary can be used for the structure\n",
+ "# from fastcore.utils import AttrDict\n",
+ "\n",
+ "def read_qmd(path):\n",
+ " \"\"\"Return notebook-like structure from a .qmd file\"\"\"\n",
+ " path = Path(path)\n",
+ " content = path.read_text(encoding='utf-8')\n",
+ "\n",
+ " cells = []\n",
+ " current_cell_source = []\n",
+ " current_cell_type = 'markdown' # Start assuming markdown\n",
+ "\n",
+ " # Regex to find fenced code blocks with language identifier\n",
+ " code_block_start_re = re.compile(r'^```{\\s*(\\w+)\\s*}')\n",
+ " code_block_end_re = re.compile(r'^```\\s*$')\n",
+ "\n",
+ " in_code_block = False\n",
+ "\n",
+ " # Iterate through lines to identify markdown and code cells\n",
+ " for line in content.splitlines(keepends=True):\n",
+ " code_match = code_block_start_re.match(line)\n",
+ " code_end_match = code_block_end_re.match(line)\n",
+ "\n",
+ " if code_match:\n",
+ " # If we were in a markdown cell, save it first\n",
+ " if current_cell_source and current_cell_type == 'markdown':\n",
+ " cells.append({\n",
+ " 'cell_type': 'markdown',\n",
+ " 'metadata': {},\n",
+ " 'source': ''.join(current_cell_source)\n",
+ " })\n",
+ " current_cell_source = []\n",
+ "\n",
+ " # Start a new code cell\n",
+ " current_cell_type = 'code'\n",
+ " in_code_block = True\n",
+ " # We skip the ````{language}` line itself in the cell source\n",
+ "\n",
+ " elif code_end_match and in_code_block:\n",
+ " # End of a code block, save the code cell\n",
+ " if current_cell_source:\n",
+ " # Remove the last line if it's just the closing fence ```\n",
+ " # (splitlines(keepends=True) includes the newline)\n",
+ " if current_cell_source[-1].strip() == '```':\n",
+ " current_cell_source.pop()\n",
+ " cells.append({\n",
+ " 'cell_type': 'code',\n",
+ " 'execution_count': None, # .qmd files don't store execution count\n",
+ " 'metadata': {}, # Metadata might be inferred later or added default\n",
+ " 'outputs': [], # .qmd files don't store outputs\n",
+ " 'source': ''.join(current_cell_source)\n",
+ " })\n",
+ " current_cell_source = []\n",
+ " current_cell_type = 'markdown' # Next content is markdown\n",
+ " in_code_block = False\n",
+ "\n",
+ " else:\n",
+ " # Add line to the current cell source\n",
+ " current_cell_source.append(line)\n",
+ "\n",
+ " # Add any remaining markdown content as a cell\n",
+ " if current_cell_source and current_cell_type == 'markdown':\n",
+ " cells.append({\n",
+ " 'cell_type': 'markdown',\n",
+ " 'metadata': {},\n",
+ " 'source': ''.join(current_cell_source)\n",
+ " })\n",
+ "\n",
+ " # Construct the final notebook-like dictionary\n",
+ " # Use default values for nbformat and metadata that mimic ipynb\n",
+ " nb_obj = {\n",
+ " 'cells': cells,\n",
+ " 'metadata': {\n",
+ " 'kernelspec': {\n",
+ " 'display_name': 'Python 3', # Default, could try to infer from ```{lang}\n",
+ " 'language': 'python', # Default\n",
+ " 'name': 'python3' # Default\n",
+ " },\n",
+ " 'language_info': { # Basic default language info\n",
+ " 'name': 'python'\n",
+ " }\n",
+ " },\n",
+ " 'nbformat': 4,\n",
+ " 'nbformat_minor': 5, # Use a recent minor version\n",
+ " 'path_': str(path)\n",
+ " }\n",
+ "\n",
+ " # If AttrDict is desired, convert here:\n",
+ " return dict2nb(nb_obj)\n",
+ " # return nb_obj\n",
+ "\n",
+ "# Example Usage (assuming minimal.qmd is in a 'tests' directory relative to execution)\n",
+ "# qmd_path = 'tests/minimal.qmd'\n",
+ "# nb_like_object = read_qmd(qmd_path)\n",
+ "# print(json.dumps(nb_like_object, indent=2)) # Print to see the structure"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "82a9830b",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "qmd_minimal = read_qmd(\"../../tests/hamux_index.qmd\")"
]
},
{
@@ -353,6 +474,77 @@
"def _is_direc(f): return getattr(f, '__name__', '-')[-1]=='_'"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "91a54352",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#|export\n",
+ "def _read_nb_or_qmd(path):\n",
+ " if Path(path).suffix == '.qmd': return read_qmd(path)\n",
+ " return read_nb(path)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b613908f",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/markdown": [
+ "```json\n",
+ "{ 'cells': [ { 'cell_type': 'markdown',\n",
+ " 'idx_': 0,\n",
+ " 'metadata': {},\n",
+ " 'source': '## A minimal notebook\\n\\n\\n'},\n",
+ " { 'cell_type': 'code',\n",
+ " 'execution_count': None,\n",
+ " 'idx_': 1,\n",
+ " 'metadata': {},\n",
+ " 'outputs': [],\n",
+ " 'source': '# Do some arithmetic\\n1+1\\n'}],\n",
+ " 'metadata': { 'kernelspec': { 'display_name': 'Python 3',\n",
+ " 'language': 'python',\n",
+ " 'name': 'python3'},\n",
+ " 'language_info': {'name': 'python'}},\n",
+ " 'nbformat': 4,\n",
+ " 'nbformat_minor': 5,\n",
+ " 'path_': '../../tests/minimal.qmd'}\n",
+ "```"
+ ],
+ "text/plain": [
+ "{'cells': [{'cell_type': 'markdown',\n",
+ " 'metadata': {},\n",
+ " 'source': '## A minimal notebook\\n\\n\\n',\n",
+ " 'idx_': 0},\n",
+ " {'cell_type': 'code',\n",
+ " 'execution_count': None,\n",
+ " 'metadata': {},\n",
+ " 'outputs': [],\n",
+ " 'source': '# Do some arithmetic\\n1+1\\n',\n",
+ " 'idx_': 1}],\n",
+ " 'metadata': {'kernelspec': {'display_name': 'Python 3',\n",
+ " 'language': 'python',\n",
+ " 'name': 'python3'},\n",
+ " 'language_info': {'name': 'python'}},\n",
+ " 'nbformat': 4,\n",
+ " 'nbformat_minor': 5,\n",
+ " 'path_': '../../tests/minimal.qmd'}"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "_read_nb_or_qmd('../../tests/minimal.qmd')"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
@@ -364,7 +556,7 @@
"class NBProcessor:\n",
" \"Process cells and nbdev comments in a notebook\"\n",
" def __init__(self, path=None, procs=None, nb=None, debug=False, rm_directives=True, process=False):\n",
- " self.nb = read_nb(path) if nb is None else nb\n",
+ " self.nb = _read_nb_or_qmd(path) if nb is None else nb\n",
" self.lang = nb_lang(self.nb)\n",
" for cell in self.nb.cells: cell.directives_ = extract_directives(cell, remove=rm_directives, lang=self.lang)\n",
" self.procs = _mk_procs(procs, nb=self.nb)\n",
@@ -439,6 +631,18 @@
"NBProcessor(everything_fn, print_execs).process()"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1645c63d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "everything_fn_qmd = '../../tests/01_everything.qmd'\n",
+ "\n",
+ "NBProcessor(everything_fn_qmd, print_execs).process()"
+ ]
+ },
{
"cell_type": "markdown",
"id": "a8202589",
@@ -641,21 +845,13 @@
"g = exec_new('import nbdev.process')\n",
"assert hasattr(g['nbdev'].process, 'NBProcessor')"
]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "af6db102-2447-49ba-94d9-bfebfb48c0f0",
- "metadata": {},
- "outputs": [],
- "source": []
}
],
"metadata": {
"kernelspec": {
- "display_name": "python3",
+ "display_name": "nbdev",
"language": "python",
- "name": "python3"
+ "name": "nbdev"
}
},
"nbformat": 4,
diff --git a/tests/01_everything.ipynb b/tests/01_everything.ipynb
index e6738ca97..e905a7587 100644
--- a/tests/01_everything.ipynb
+++ b/tests/01_everything.ipynb
@@ -29,7 +29,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|export\n",
@@ -39,7 +43,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|export\n",
@@ -50,7 +58,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|hide\n",
@@ -68,7 +80,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"import nbdev.x,y,nbdev.z\n",
@@ -78,7 +94,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|export\n",
@@ -88,7 +108,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|export\n",
@@ -101,7 +125,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|hide\n",
@@ -112,7 +140,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|exporti\n",
@@ -123,7 +155,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|export\n",
@@ -138,7 +174,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|export\n",
@@ -149,7 +189,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|export\n",
@@ -160,7 +204,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|export\n",
@@ -170,7 +218,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|export some.thing\n",
@@ -180,7 +232,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"def i_n(): ..."
@@ -189,7 +245,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"def j_n(): ...\n",
@@ -199,7 +259,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#export is used in a full sentence here\n",
@@ -210,7 +274,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#exporting\n",
@@ -220,7 +288,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|export\n",
@@ -230,7 +302,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|export\n",
@@ -242,7 +318,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|export\n",
@@ -252,7 +332,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|export\n",
@@ -262,7 +346,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": [
"#|printme testing\n",
@@ -272,7 +360,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [
{
"data": {
@@ -295,7 +387,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [
{
"name": "stdout",
@@ -312,7 +408,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [
{
"name": "stdout",
@@ -330,7 +430,11 @@
{
"cell_type": "code",
"execution_count": null,
- "metadata": {},
+ "metadata": {
+ "vscode": {
+ "languageId": "python"
+ }
+ },
"outputs": [],
"source": []
}
diff --git a/tests/01_everything.qmd b/tests/01_everything.qmd
new file mode 100644
index 000000000..049ceecb6
--- /dev/null
+++ b/tests/01_everything.qmd
@@ -0,0 +1,183 @@
+---
+title: Foo
+execute:
+ echo: false
+---
+
+# A Title
+
+> A description
+
+This notebook is used to demonstrate and test all the features of nbdev's export functionality. See the notebooks in `nbs` for how it's used.
+
+
+```{python}
+#|export
+from __future__ import print_function
+```
+
+
+```{python}
+#|export
+from __future__ import absolute_import
+from fastcore.utils import patch,patch_to
+```
+
+
+```{python}
+#|hide
+#|default_exp everything
+#|default_cls_lvl 3
+```
+
+Each symbol name below has >=2 parts, split on `_`. First part is a unique name. Second is `y` if it should be exported. Third is `nall` if it shouldn't appear in `__all__`.
+
+
+```{python}
+import nbdev.x,y,nbdev.z
+from nbdev.x import *
+```
+
+
+```{python}
+#|export
+def a_y(): ...
+```
+
+
+```{python}
+#|export
+def a_y(f): # should only appear once
+ def ai_n(): ...
+ class a2i_n(): ...
+ return f
+```
+
+
+```{python}
+#|hide
+#|export
+def b_y(a:bool)->int: ...
+```
+
+
+```{python}
+#|exporti
+#just another comment
+def c_y_nall(): ...
+```
+
+
+```{python}
+#|export
+class d_y():
+ def di_n(): ...
+ class d2i_n(): ...
+
+@a_y
+async def e_y(): ...
+```
+
+
+```{python}
+#|export
+@patch
+def d3i_n(self:d_y): ...
+```
+
+
+```{python}
+#|export
+@patch_to(d_y)
+def d4i_n(self): ...
+```
+
+
+```{python}
+#|export
+def _f_y_nall(): ...
+```
+
+
+```{python}
+#|export some.thing
+def h_n(): ...
+```
+
+
+```{python}
+def i_n(): ...
+```
+
+
+```{python}
+def j_n(): ...
+#|export
+```
+
+
+```{python}
+#export is used in a full sentence here
+#therefore this is not exported
+def k_n(): ...
+```
+
+
+```{python}
+#exporting
+def l_n(): ...
+```
+
+
+```{python}
+#|export
+m_y = n_y = 1
+```
+
+
+```{python}
+#|export
+exec("o_y=1")
+exec("p_y=1")
+_all_ = [o_y, 'p_y']
+```
+
+
+```{python}
+#|export
+q_y,_r_n = (1,1)
+```
+
+
+```{python}
+#|export
+a_y.test = 1
+```
+
+
+```{python}
+#|printme testing
+_tmp = "Cell for testing processor subclass"
+```
+
+
+```{python}
+%%html
+a test
+```
+
+
+```{python}
+print('\033[94mhello')
+```
+
+
+```{python}
+#|eval:false
+print("do not print me!")
+```
+
+
+```{python}
+
+```
\ No newline at end of file
diff --git a/tests/minimal.qmd b/tests/minimal.qmd
new file mode 100644
index 000000000..3c46a4551
--- /dev/null
+++ b/tests/minimal.qmd
@@ -0,0 +1,7 @@
+## A minimal notebook
+
+
+```{python}
+# Do some arithmetic
+1+1
+```
\ No newline at end of file
From bf8c091fec8b3b48c2494e091920af0e1ca3ce3e Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Wed, 28 May 2025 09:43:07 -0400
Subject: [PATCH 02/31] Fix file glob for nbdev to detect .qmd files as source
nbs
---
nbdev/clean.py | 3 ++-
nbdev/doclinks.py | 10 ++++----
nbs/api/03_process.ipynb | 51 +++++++++++++++++++++++++++++----------
nbs/api/05_doclinks.ipynb | 47 ++++++++++++++++++++++++++----------
nbs/api/11_clean.ipynb | 14 +++++++++--
5 files changed, 91 insertions(+), 34 deletions(-)
diff --git a/nbdev/clean.py b/nbdev/clean.py
index 9bf06848b..b971fdc4c 100644
--- a/nbdev/clean.py
+++ b/nbdev/clean.py
@@ -128,6 +128,7 @@ def _nbdev_clean(nb, path=None, clear_all=None):
if path: nbdev_trust.__wrapped__(path)
# %% ../nbs/api/11_clean.ipynb
+#|export
@call_parse
def nbdev_clean(
fname:str=None, # A notebook name or glob to clean
@@ -141,7 +142,7 @@ def nbdev_clean(
_write = partial(process_write, warn_msg='Failed to clean notebook', proc_nb=_clean)
if stdin: return _write(f_in=sys.stdin, f_out=sys.stdout)
if fname is None: fname = get_config().nbs_path
- for f in globtastic(fname, file_glob='*.ipynb', skip_folder_re='^[_.]'): _write(f_in=f, disp=disp)
+ for f in globtastic(fname, file_re=r'.*\.ipynb$|.*\.qmd$', skip_folder_re='^[_.]'): _write(f_in=f, disp=disp)
# %% ../nbs/api/11_clean.ipynb
def clean_jupyter(path, model, **kwargs):
diff --git a/nbdev/doclinks.py b/nbdev/doclinks.py
index 8efbb40c7..dec68e5fd 100644
--- a/nbdev/doclinks.py
+++ b/nbdev/doclinks.py
@@ -116,20 +116,20 @@ def _build_modidx(dest=None, nbs_path=None, skip_exists=False):
# %% ../nbs/api/05_doclinks.ipynb
@delegates(globtastic)
-def nbglob(path=None, skip_folder_re = '^[_.]', file_glob='*.ipynb', skip_file_re='^[_.]', key='nbs_path', as_path=False, **kwargs):
+def nbglob(path=None, skip_folder_re = '^[_.]', file_re=r'.*\.ipynb$|.*\.qmd$', skip_file_re='^[_.]', key='nbs_path', as_path=False, **kwargs):
"Find all files in a directory matching an extension given a config key."
path = Path(path or get_config()[key])
recursive=get_config().recursive
- res = globtastic(path, file_glob=file_glob, skip_folder_re=skip_folder_re,
+ res = globtastic(path, file_re=file_re, skip_folder_re=skip_folder_re,
skip_file_re=skip_file_re, recursive=recursive, **kwargs)
return res.map(Path) if as_path else res
-
# %% ../nbs/api/05_doclinks.ipynb
+#|export
def nbglob_cli(
path:str=None, # Path to notebooks
symlinks:bool=False, # Follow symlinks?
- file_glob:str='*.ipynb', # Only include files matching glob
- file_re:str=None, # Only include files matching regex
+ file_glob:str=None, # Only include files matching glob
+ file_re:str=r'.*\.ipynb$|.*\.qmd$', # Only include files matching regex
folder_re:str=None, # Only enter folders matching regex
skip_file_glob:str=None, # Skip files matching glob
skip_file_re:str='^[_.]', # Skip files matching regex
diff --git a/nbs/api/03_process.ipynb b/nbs/api/03_process.ipynb
index 0afb5739e..7bdb4e6ae 100644
--- a/nbs/api/03_process.ipynb
+++ b/nbs/api/03_process.ipynb
@@ -183,16 +183,6 @@
"# print(json.dumps(nb_like_object, indent=2)) # Print to see the structure"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "82a9830b",
- "metadata": {},
- "outputs": [],
- "source": [
- "qmd_minimal = read_qmd(\"../../tests/hamux_index.qmd\")"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -636,13 +626,48 @@
"execution_count": null,
"id": "1645c63d",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "---\n",
+ "title: Foo\n",
+ "execute:\n",
+ " echo: false\n",
+ "---\n",
+ "\n",
+ "# A Title\n",
+ "\n",
+ "> A description\n",
+ "\n",
+ "This notebook is used to demonstrate and test all the features of nbdev's export functionality. See the notebooks in `nbs` for how it's used.\n",
+ "\n",
+ "\n",
+ "\n",
+ "exec(\"o_y=1\")\n",
+ "exec(\"p_y=1\")\n",
+ "_all_ = [o_y, 'p_y']\n",
+ "\n"
+ ]
+ }
+ ],
"source": [
"everything_fn_qmd = '../../tests/01_everything.qmd'\n",
"\n",
"NBProcessor(everything_fn_qmd, print_execs).process()"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d4772302",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "NBProcessor(everything_fn_qmd, print_execs).process()"
+ ]
+ },
{
"cell_type": "markdown",
"id": "a8202589",
@@ -849,9 +874,9 @@
],
"metadata": {
"kernelspec": {
- "display_name": "nbdev",
+ "display_name": "python3",
"language": "python",
- "name": "nbdev"
+ "name": "python3"
}
},
"nbformat": 4,
diff --git a/nbs/api/05_doclinks.ipynb b/nbs/api/05_doclinks.ipynb
index 4b4097eda..fbac97fec 100644
--- a/nbs/api/05_doclinks.ipynb
+++ b/nbs/api/05_doclinks.ipynb
@@ -321,11 +321,11 @@
"source": [
"#|export\n",
"@delegates(globtastic)\n",
- "def nbglob(path=None, skip_folder_re = '^[_.]', file_glob='*.ipynb', skip_file_re='^[_.]', key='nbs_path', as_path=False, **kwargs):\n",
+ "def nbglob(path=None, skip_folder_re = '^[_.]', file_re=r'.*\\.ipynb|.*\\.qmd', skip_file_re='^[_.]', key='nbs_path', as_path=False, **kwargs):\n",
" \"Find all files in a directory matching an extension given a config key.\"\n",
" path = Path(path or get_config()[key])\n",
" recursive=get_config().recursive\n",
- " res = globtastic(path, file_glob=file_glob, skip_folder_re=skip_folder_re,\n",
+ " res = globtastic(path, file_re=file_re, skip_folder_re=skip_folder_re,\n",
" skip_file_re=skip_file_re, recursive=recursive, **kwargs)\n",
" return res.map(Path) if as_path else res"
]
@@ -340,8 +340,8 @@
"def nbglob_cli(\n",
" path:str=None, # Path to notebooks\n",
" symlinks:bool=False, # Follow symlinks?\n",
- " file_glob:str='*.ipynb', # Only include files matching glob\n",
- " file_re:str=None, # Only include files matching regex\n",
+ " file_glob:str=None, # Only include files matching glob\n",
+ " file_re:str=r'.*\\.ipynb|.*\\.qmd', # Only include files matching regex\n",
" folder_re:str=None, # Only enter folders matching regex\n",
" skip_file_glob:str=None, # Skip files matching glob\n",
" skip_file_re:str='^[_.]', # Skip files matching regex\n",
@@ -784,7 +784,7 @@
"text/plain": [
"('https://nbdev.fast.ai/api/doclinks.html#nbdevlookup',\n",
" 'nbdev/doclinks.py',\n",
- " 'https://github.com/AnswerDotAI/nbdev/blob/master/nbdev/doclinks.py')"
+ " 'https://github.com/AnswerDotAI/nbdev/blob/main/nbdev/doclinks.py')"
]
},
"execution_count": null,
@@ -807,7 +807,7 @@
"text/markdown": [
"---\n",
"\n",
- "[source](https://github.com/AnswerDotAI/nbdev/blob/master/nbdev/doclinks.py#L269){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+ "[source](https://github.com/AnswerDotAI/nbdev/blob/main/nbdev/doclinks.py#L269){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.doc\n",
"\n",
@@ -818,7 +818,7 @@
"text/plain": [
"---\n",
"\n",
- "[source](https://github.com/AnswerDotAI/nbdev/blob/master/nbdev/doclinks.py#L269){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+ "[source](https://github.com/AnswerDotAI/nbdev/blob/main/nbdev/doclinks.py#L269){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.doc\n",
"\n",
@@ -909,7 +909,7 @@
"text/markdown": [
"---\n",
"\n",
- "[source](https://github.com/AnswerDotAI/nbdev/blob/master/nbdev/doclinks.py#L274){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+ "[source](https://github.com/AnswerDotAI/nbdev/blob/main/nbdev/doclinks.py#L274){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.code\n",
"\n",
@@ -920,7 +920,7 @@
"text/plain": [
"---\n",
"\n",
- "[source](https://github.com/AnswerDotAI/nbdev/blob/master/nbdev/doclinks.py#L274){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+ "[source](https://github.com/AnswerDotAI/nbdev/blob/main/nbdev/doclinks.py#L274){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.code\n",
"\n",
@@ -946,7 +946,7 @@
{
"data": {
"text/plain": [
- "'https://github.com/AnswerDotAI/fastcore/blob/master/fastcore/net.py#LNone'"
+ "'https://github.com/AnswerDotAI/fastcore/blob/main/fastcore/net.py#LNone'"
]
},
"execution_count": null,
@@ -968,7 +968,7 @@
"text/markdown": [
"---\n",
"\n",
- "[source](https://github.com/AnswerDotAI/nbdev/blob/master/nbdev/doclinks.py#L292){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+ "[source](https://github.com/AnswerDotAI/nbdev/blob/main/nbdev/doclinks.py#L292){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.linkify\n",
"\n",
@@ -977,7 +977,7 @@
"text/plain": [
"---\n",
"\n",
- "[source](https://github.com/AnswerDotAI/nbdev/blob/master/nbdev/doclinks.py#L292){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
+ "[source](https://github.com/AnswerDotAI/nbdev/blob/main/nbdev/doclinks.py#L292){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n",
"\n",
"### NbdevLookup.linkify\n",
"\n",
@@ -1216,7 +1216,28 @@
"cell_type": "code",
"execution_count": null,
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "ename": "JSONDecodeError",
+ "evalue": "Extra data: line 1 column 3 (char 2)",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mJSONDecodeError\u001b[0m Traceback (most recent call last)",
+ "Cell \u001b[0;32mIn[54], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m#|eval: false\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m#|hide\u001b[39;00m\n\u001b[1;32m 3\u001b[0m Path(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m../nbdev/export.py\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;241m.\u001b[39munlink(missing_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m----> 4\u001b[0m \u001b[43mnbdev_export\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 6\u001b[0m g \u001b[38;5;241m=\u001b[39m exec_new(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mimport nbdev.export\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 7\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(g[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnbdev\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mexport, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnb_export\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
+ "File \u001b[0;32m~/miniconda3/envs/nbdev/lib/python3.10/site-packages/fastcore/script.py:116\u001b[0m, in \u001b[0;36mcall_parse.._f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[38;5;129m@wraps\u001b[39m(func)\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m_f\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 115\u001b[0m mod \u001b[38;5;241m=\u001b[39m inspect\u001b[38;5;241m.\u001b[39mgetmodule(inspect\u001b[38;5;241m.\u001b[39mcurrentframe()\u001b[38;5;241m.\u001b[39mf_back)\n\u001b[0;32m--> 116\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m mod: \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 117\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m SCRIPT_INFO\u001b[38;5;241m.\u001b[39mfunc \u001b[38;5;129;01mand\u001b[39;00m mod\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;241m==\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m\"\u001b[39m: SCRIPT_INFO\u001b[38;5;241m.\u001b[39mfunc \u001b[38;5;241m=\u001b[39m func\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\n\u001b[1;32m 118\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(sys\u001b[38;5;241m.\u001b[39margv)\u001b[38;5;241m>\u001b[39m\u001b[38;5;241m1\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m sys\u001b[38;5;241m.\u001b[39margv[\u001b[38;5;241m1\u001b[39m]\u001b[38;5;241m==\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m: sys\u001b[38;5;241m.\u001b[39margv\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;241m1\u001b[39m)\n",
+ "Cell \u001b[0;32mIn[20], line 15\u001b[0m, in \u001b[0;36mnbdev_export\u001b[0;34m(path, procs, **kwargs)\u001b[0m\n\u001b[1;32m 13\u001b[0m procs \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mgetattr\u001b[39m(nbdev\u001b[38;5;241m.\u001b[39mexport, p) \u001b[38;5;28;01mfor\u001b[39;00m p \u001b[38;5;129;01min\u001b[39;00m L(procs)]\n\u001b[1;32m 14\u001b[0m files \u001b[38;5;241m=\u001b[39m nbglob(path\u001b[38;5;241m=\u001b[39mpath, as_path\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\u001b[38;5;241m.\u001b[39msorted(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 15\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m files: \u001b[43mnb_export\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprocs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprocs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 16\u001b[0m add_init(get_config()\u001b[38;5;241m.\u001b[39mlib_path)\n\u001b[1;32m 17\u001b[0m _build_modidx()\n",
+ "File \u001b[0;32m~/Projects/nbdev/nbdev/export.py:81\u001b[0m, in \u001b[0;36mnb_export\u001b[0;34m(nbname, lib_path, procs, name, mod_maker, debug, solo_nb)\u001b[0m\n\u001b[1;32m 79\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m lib_path \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m: lib_path \u001b[38;5;241m=\u001b[39m get_config()\u001b[38;5;241m.\u001b[39mlib_path \u001b[38;5;28;01mif\u001b[39;00m is_nbdev() \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 80\u001b[0m exp \u001b[38;5;241m=\u001b[39m ExportModuleProc()\n\u001b[0;32m---> 81\u001b[0m nb \u001b[38;5;241m=\u001b[39m \u001b[43mNBProcessor\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\u001b[43mexp\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43mL\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdebug\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdebug\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 82\u001b[0m nb\u001b[38;5;241m.\u001b[39mprocess()\n\u001b[1;32m 83\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m mod,cells \u001b[38;5;129;01min\u001b[39;00m exp\u001b[38;5;241m.\u001b[39mmodules\u001b[38;5;241m.\u001b[39mitems():\n",
+ "File \u001b[0;32m~/Projects/nbdev/nbdev/process.py:204\u001b[0m, in \u001b[0;36mNBProcessor.__init__\u001b[0;34m(self, path, procs, nb, debug, rm_directives, process)\u001b[0m\n\u001b[1;32m 203\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, path\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, procs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, nb\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, debug\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, rm_directives\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, process\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[0;32m--> 204\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnb \u001b[38;5;241m=\u001b[39m \u001b[43m_read_nb_or_qmd\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m nb \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m nb\n\u001b[1;32m 205\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlang \u001b[38;5;241m=\u001b[39m nb_lang(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnb)\n\u001b[1;32m 206\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m cell \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnb\u001b[38;5;241m.\u001b[39mcells: cell\u001b[38;5;241m.\u001b[39mdirectives_ \u001b[38;5;241m=\u001b[39m extract_directives(cell, remove\u001b[38;5;241m=\u001b[39mrm_directives, lang\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlang)\n",
+ "File \u001b[0;32m~/Projects/nbdev/nbdev/process.py:198\u001b[0m, in \u001b[0;36m_read_nb_or_qmd\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m 196\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m_read_nb_or_qmd\u001b[39m(path):\n\u001b[1;32m 197\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m Path(path)\u001b[38;5;241m.\u001b[39msuffix \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.qmd\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;28;01mreturn\u001b[39;00m read_qmd(path)\n\u001b[0;32m--> 198\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mread_nb\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n",
+ "File \u001b[0;32m~/miniconda3/envs/nbdev/lib/python3.10/site-packages/execnb/nbio.py:59\u001b[0m, in \u001b[0;36mread_nb\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21mread_nb\u001b[39m(path):\n\u001b[1;32m 58\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mReturn notebook at `path`\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m---> 59\u001b[0m res \u001b[38;5;241m=\u001b[39m dict2nb(\u001b[43m_read_json\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mutf-8\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m)\n\u001b[1;32m 60\u001b[0m res[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpath_\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(path)\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m res\n",
+ "File \u001b[0;32m~/miniconda3/envs/nbdev/lib/python3.10/site-packages/execnb/nbio.py:18\u001b[0m, in \u001b[0;36m_read_json\u001b[0;34m(self, encoding, errors)\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m_read_json\u001b[39m(\u001b[38;5;28mself\u001b[39m, encoding\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, errors\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m---> 18\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mloads\u001b[49m\u001b[43m(\u001b[49m\u001b[43mPath\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_text\u001b[49m\u001b[43m(\u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoding\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merrors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43merrors\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
+ "File \u001b[0;32m~/miniconda3/envs/nbdev/lib/python3.10/json/__init__.py:346\u001b[0m, in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m 341\u001b[0m s \u001b[38;5;241m=\u001b[39m s\u001b[38;5;241m.\u001b[39mdecode(detect_encoding(s), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msurrogatepass\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m 344\u001b[0m parse_int \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m parse_float \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m 345\u001b[0m parse_constant \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_pairs_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m kw):\n\u001b[0;32m--> 346\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_default_decoder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 347\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 348\u001b[0m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;241m=\u001b[39m JSONDecoder\n",
+ "File \u001b[0;32m~/miniconda3/envs/nbdev/lib/python3.10/json/decoder.py:340\u001b[0m, in \u001b[0;36mJSONDecoder.decode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m 338\u001b[0m end \u001b[38;5;241m=\u001b[39m _w(s, end)\u001b[38;5;241m.\u001b[39mend()\n\u001b[1;32m 339\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m end \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mlen\u001b[39m(s):\n\u001b[0;32m--> 340\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m JSONDecodeError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExtra data\u001b[39m\u001b[38;5;124m\"\u001b[39m, s, end)\n\u001b[1;32m 341\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m obj\n",
+ "\u001b[0;31mJSONDecodeError\u001b[0m: Extra data: line 1 column 3 (char 2)"
+ ]
+ }
+ ],
"source": [
"#|eval: false\n",
"#|hide\n",
diff --git a/nbs/api/11_clean.ipynb b/nbs/api/11_clean.ipynb
index 5e88ca73b..3d983a266 100644
--- a/nbs/api/11_clean.ipynb
+++ b/nbs/api/11_clean.ipynb
@@ -88,7 +88,7 @@
" path = fname if fname.is_dir() else fname.parent\n",
" check_fname = path/\".last_checked\"\n",
" last_checked = os.path.getmtime(check_fname) if check_fname.exists() else None\n",
- " nbs = globtastic(fname, file_glob='*.ipynb', skip_folder_re='^[_.]') if fname.is_dir() else [fname]\n",
+ " nbs = globtastic(fname, file_re=r'.*\\.ipynb$|.*\\.qmd$', skip_folder_re='^[_.]') if fname.is_dir() else [fname]\n",
" for fn in nbs:\n",
" if last_checked and not force_all:\n",
" last_changed = os.path.getmtime(fn)\n",
@@ -399,7 +399,17 @@
" _write = partial(process_write, warn_msg='Failed to clean notebook', proc_nb=_clean)\n",
" if stdin: return _write(f_in=sys.stdin, f_out=sys.stdout)\n",
" if fname is None: fname = get_config().nbs_path\n",
- " for f in globtastic(fname, file_glob='*.ipynb', skip_folder_re='^[_.]'): _write(f_in=f, disp=disp)"
+ " for f in globtastic(fname, file_re=r'.*\\.ipynb$|.*\\.qmd$', skip_folder_re='^[_.]'): _write(f_in=f, disp=disp)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "pat = re.compile(r'.*\\.ipynb$|.*\\.qmd$')\n",
+ "if pat.search('123.qmd.py'): print('yes')"
]
},
{
From fceff8fb4b27d13caf819ffcaf0f4f3d2e885c84 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Wed, 28 May 2025 10:51:13 -0400
Subject: [PATCH 03/31] Don't clean .qmd files
---
nbdev/clean.py | 5 ++--
nbdev/doclinks.py | 2 +-
nbs/api/03_process.ipynb | 52 ++-------------------------------------
nbs/api/05_doclinks.ipynb | 27 +++-----------------
nbs/api/11_clean.ipynb | 2 +-
5 files changed, 9 insertions(+), 79 deletions(-)
diff --git a/nbdev/clean.py b/nbdev/clean.py
index b971fdc4c..9ff0ec693 100644
--- a/nbdev/clean.py
+++ b/nbdev/clean.py
@@ -38,7 +38,7 @@ def nbdev_trust(
path = fname if fname.is_dir() else fname.parent
check_fname = path/".last_checked"
last_checked = os.path.getmtime(check_fname) if check_fname.exists() else None
- nbs = globtastic(fname, file_glob='*.ipynb', skip_folder_re='^[_.]') if fname.is_dir() else [fname]
+ nbs = globtastic(fname, file_re=r'.*\.ipynb$|.*\.qmd$', skip_folder_re='^[_.]') if fname.is_dir() else [fname]
for fn in nbs:
if last_checked and not force_all:
last_changed = os.path.getmtime(fn)
@@ -128,7 +128,6 @@ def _nbdev_clean(nb, path=None, clear_all=None):
if path: nbdev_trust.__wrapped__(path)
# %% ../nbs/api/11_clean.ipynb
-#|export
@call_parse
def nbdev_clean(
fname:str=None, # A notebook name or glob to clean
@@ -142,7 +141,7 @@ def nbdev_clean(
_write = partial(process_write, warn_msg='Failed to clean notebook', proc_nb=_clean)
if stdin: return _write(f_in=sys.stdin, f_out=sys.stdout)
if fname is None: fname = get_config().nbs_path
- for f in globtastic(fname, file_re=r'.*\.ipynb$|.*\.qmd$', skip_folder_re='^[_.]'): _write(f_in=f, disp=disp)
+ for f in globtastic(fname, file_re=r'.*\.ipynb$', skip_folder_re='^[_.]'): _write(f_in=f, disp=disp) # Don't clean .qmd files
# %% ../nbs/api/11_clean.ipynb
def clean_jupyter(path, model, **kwargs):
diff --git a/nbdev/doclinks.py b/nbdev/doclinks.py
index dec68e5fd..cb6ae4aa8 100644
--- a/nbdev/doclinks.py
+++ b/nbdev/doclinks.py
@@ -123,8 +123,8 @@ def nbglob(path=None, skip_folder_re = '^[_.]', file_re=r'.*\.ipynb$|.*\.qmd$',
res = globtastic(path, file_re=file_re, skip_folder_re=skip_folder_re,
skip_file_re=skip_file_re, recursive=recursive, **kwargs)
return res.map(Path) if as_path else res
+
# %% ../nbs/api/05_doclinks.ipynb
-#|export
def nbglob_cli(
path:str=None, # Path to notebooks
symlinks:bool=False, # Follow symlinks?
diff --git a/nbs/api/03_process.ipynb b/nbs/api/03_process.ipynb
index 7bdb4e6ae..a7ed8fd2e 100644
--- a/nbs/api/03_process.ipynb
+++ b/nbs/api/03_process.ipynb
@@ -482,57 +482,9 @@
"execution_count": null,
"id": "b613908f",
"metadata": {},
- "outputs": [
- {
- "data": {
- "text/markdown": [
- "```json\n",
- "{ 'cells': [ { 'cell_type': 'markdown',\n",
- " 'idx_': 0,\n",
- " 'metadata': {},\n",
- " 'source': '## A minimal notebook\\n\\n\\n'},\n",
- " { 'cell_type': 'code',\n",
- " 'execution_count': None,\n",
- " 'idx_': 1,\n",
- " 'metadata': {},\n",
- " 'outputs': [],\n",
- " 'source': '# Do some arithmetic\\n1+1\\n'}],\n",
- " 'metadata': { 'kernelspec': { 'display_name': 'Python 3',\n",
- " 'language': 'python',\n",
- " 'name': 'python3'},\n",
- " 'language_info': {'name': 'python'}},\n",
- " 'nbformat': 4,\n",
- " 'nbformat_minor': 5,\n",
- " 'path_': '../../tests/minimal.qmd'}\n",
- "```"
- ],
- "text/plain": [
- "{'cells': [{'cell_type': 'markdown',\n",
- " 'metadata': {},\n",
- " 'source': '## A minimal notebook\\n\\n\\n',\n",
- " 'idx_': 0},\n",
- " {'cell_type': 'code',\n",
- " 'execution_count': None,\n",
- " 'metadata': {},\n",
- " 'outputs': [],\n",
- " 'source': '# Do some arithmetic\\n1+1\\n',\n",
- " 'idx_': 1}],\n",
- " 'metadata': {'kernelspec': {'display_name': 'Python 3',\n",
- " 'language': 'python',\n",
- " 'name': 'python3'},\n",
- " 'language_info': {'name': 'python'}},\n",
- " 'nbformat': 4,\n",
- " 'nbformat_minor': 5,\n",
- " 'path_': '../../tests/minimal.qmd'}"
- ]
- },
- "execution_count": null,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
- "_read_nb_or_qmd('../../tests/minimal.qmd')"
+ "out = _read_nb_or_qmd('../../tests/minimal.qmd')"
]
},
{
diff --git a/nbs/api/05_doclinks.ipynb b/nbs/api/05_doclinks.ipynb
index fbac97fec..82f81af9b 100644
--- a/nbs/api/05_doclinks.ipynb
+++ b/nbs/api/05_doclinks.ipynb
@@ -321,7 +321,7 @@
"source": [
"#|export\n",
"@delegates(globtastic)\n",
- "def nbglob(path=None, skip_folder_re = '^[_.]', file_re=r'.*\\.ipynb|.*\\.qmd', skip_file_re='^[_.]', key='nbs_path', as_path=False, **kwargs):\n",
+ "def nbglob(path=None, skip_folder_re = '^[_.]', file_re=r'.*\\.ipynb$|.*\\.qmd$', skip_file_re='^[_.]', key='nbs_path', as_path=False, **kwargs):\n",
" \"Find all files in a directory matching an extension given a config key.\"\n",
" path = Path(path or get_config()[key])\n",
" recursive=get_config().recursive\n",
@@ -341,7 +341,7 @@
" path:str=None, # Path to notebooks\n",
" symlinks:bool=False, # Follow symlinks?\n",
" file_glob:str=None, # Only include files matching glob\n",
- " file_re:str=r'.*\\.ipynb|.*\\.qmd', # Only include files matching regex\n",
+ " file_re:str=r'.*\\.ipynb$|.*\\.qmd$', # Only include files matching regex\n",
" folder_re:str=None, # Only enter folders matching regex\n",
" skip_file_glob:str=None, # Skip files matching glob\n",
" skip_file_re:str='^[_.]', # Skip files matching regex\n",
@@ -1216,28 +1216,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {},
- "outputs": [
- {
- "ename": "JSONDecodeError",
- "evalue": "Extra data: line 1 column 3 (char 2)",
- "output_type": "error",
- "traceback": [
- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
- "\u001b[0;31mJSONDecodeError\u001b[0m Traceback (most recent call last)",
- "Cell \u001b[0;32mIn[54], line 4\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m#|eval: false\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m#|hide\u001b[39;00m\n\u001b[1;32m 3\u001b[0m Path(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m../nbdev/export.py\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;241m.\u001b[39munlink(missing_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m----> 4\u001b[0m \u001b[43mnbdev_export\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 6\u001b[0m g \u001b[38;5;241m=\u001b[39m exec_new(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mimport nbdev.export\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 7\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(g[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnbdev\u001b[39m\u001b[38;5;124m'\u001b[39m]\u001b[38;5;241m.\u001b[39mexport, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mnb_export\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
- "File \u001b[0;32m~/miniconda3/envs/nbdev/lib/python3.10/site-packages/fastcore/script.py:116\u001b[0m, in \u001b[0;36mcall_parse.._f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 113\u001b[0m \u001b[38;5;129m@wraps\u001b[39m(func)\n\u001b[1;32m 114\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m_f\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m 115\u001b[0m mod \u001b[38;5;241m=\u001b[39m inspect\u001b[38;5;241m.\u001b[39mgetmodule(inspect\u001b[38;5;241m.\u001b[39mcurrentframe()\u001b[38;5;241m.\u001b[39mf_back)\n\u001b[0;32m--> 116\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m mod: \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 117\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m SCRIPT_INFO\u001b[38;5;241m.\u001b[39mfunc \u001b[38;5;129;01mand\u001b[39;00m mod\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\u001b[38;5;241m==\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m\"\u001b[39m: SCRIPT_INFO\u001b[38;5;241m.\u001b[39mfunc \u001b[38;5;241m=\u001b[39m func\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\n\u001b[1;32m 118\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(sys\u001b[38;5;241m.\u001b[39margv)\u001b[38;5;241m>\u001b[39m\u001b[38;5;241m1\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m sys\u001b[38;5;241m.\u001b[39margv[\u001b[38;5;241m1\u001b[39m]\u001b[38;5;241m==\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m'\u001b[39m: sys\u001b[38;5;241m.\u001b[39margv\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;241m1\u001b[39m)\n",
- "Cell \u001b[0;32mIn[20], line 15\u001b[0m, in \u001b[0;36mnbdev_export\u001b[0;34m(path, procs, **kwargs)\u001b[0m\n\u001b[1;32m 13\u001b[0m procs \u001b[38;5;241m=\u001b[39m [\u001b[38;5;28mgetattr\u001b[39m(nbdev\u001b[38;5;241m.\u001b[39mexport, p) \u001b[38;5;28;01mfor\u001b[39;00m p \u001b[38;5;129;01min\u001b[39;00m L(procs)]\n\u001b[1;32m 14\u001b[0m files \u001b[38;5;241m=\u001b[39m nbglob(path\u001b[38;5;241m=\u001b[39mpath, as_path\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\u001b[38;5;241m.\u001b[39msorted(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m---> 15\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m files: \u001b[43mnb_export\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mprocs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mprocs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 16\u001b[0m add_init(get_config()\u001b[38;5;241m.\u001b[39mlib_path)\n\u001b[1;32m 17\u001b[0m _build_modidx()\n",
- "File \u001b[0;32m~/Projects/nbdev/nbdev/export.py:81\u001b[0m, in \u001b[0;36mnb_export\u001b[0;34m(nbname, lib_path, procs, name, mod_maker, debug, solo_nb)\u001b[0m\n\u001b[1;32m 79\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m lib_path \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m: lib_path \u001b[38;5;241m=\u001b[39m get_config()\u001b[38;5;241m.\u001b[39mlib_path \u001b[38;5;28;01mif\u001b[39;00m is_nbdev() \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m'\u001b[39m\n\u001b[1;32m 80\u001b[0m exp \u001b[38;5;241m=\u001b[39m ExportModuleProc()\n\u001b[0;32m---> 81\u001b[0m nb \u001b[38;5;241m=\u001b[39m \u001b[43mNBProcessor\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnbname\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m[\u001b[49m\u001b[43mexp\u001b[49m\u001b[43m]\u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43mL\u001b[49m\u001b[43m(\u001b[49m\u001b[43mprocs\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdebug\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdebug\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 82\u001b[0m nb\u001b[38;5;241m.\u001b[39mprocess()\n\u001b[1;32m 83\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m mod,cells \u001b[38;5;129;01min\u001b[39;00m exp\u001b[38;5;241m.\u001b[39mmodules\u001b[38;5;241m.\u001b[39mitems():\n",
- "File \u001b[0;32m~/Projects/nbdev/nbdev/process.py:204\u001b[0m, in \u001b[0;36mNBProcessor.__init__\u001b[0;34m(self, path, procs, nb, debug, rm_directives, process)\u001b[0m\n\u001b[1;32m 203\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, path\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, procs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, nb\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, debug\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m, rm_directives\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, process\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m):\n\u001b[0;32m--> 204\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnb \u001b[38;5;241m=\u001b[39m \u001b[43m_read_nb_or_qmd\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mif\u001b[39;00m nb \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m nb\n\u001b[1;32m 205\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlang \u001b[38;5;241m=\u001b[39m nb_lang(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnb)\n\u001b[1;32m 206\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m cell \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnb\u001b[38;5;241m.\u001b[39mcells: cell\u001b[38;5;241m.\u001b[39mdirectives_ \u001b[38;5;241m=\u001b[39m extract_directives(cell, remove\u001b[38;5;241m=\u001b[39mrm_directives, lang\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlang)\n",
- "File \u001b[0;32m~/Projects/nbdev/nbdev/process.py:198\u001b[0m, in \u001b[0;36m_read_nb_or_qmd\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m 196\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m_read_nb_or_qmd\u001b[39m(path):\n\u001b[1;32m 197\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m Path(path)\u001b[38;5;241m.\u001b[39msuffix \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m.qmd\u001b[39m\u001b[38;5;124m'\u001b[39m: \u001b[38;5;28;01mreturn\u001b[39;00m read_qmd(path)\n\u001b[0;32m--> 198\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mread_nb\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m)\u001b[49m\n",
- "File \u001b[0;32m~/miniconda3/envs/nbdev/lib/python3.10/site-packages/execnb/nbio.py:59\u001b[0m, in \u001b[0;36mread_nb\u001b[0;34m(path)\u001b[0m\n\u001b[1;32m 57\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21mread_nb\u001b[39m(path):\n\u001b[1;32m 58\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mReturn notebook at `path`\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m---> 59\u001b[0m res \u001b[38;5;241m=\u001b[39m dict2nb(\u001b[43m_read_json\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mutf-8\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m)\n\u001b[1;32m 60\u001b[0m res[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpath_\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mstr\u001b[39m(path)\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m res\n",
- "File \u001b[0;32m~/miniconda3/envs/nbdev/lib/python3.10/site-packages/execnb/nbio.py:18\u001b[0m, in \u001b[0;36m_read_json\u001b[0;34m(self, encoding, errors)\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m_read_json\u001b[39m(\u001b[38;5;28mself\u001b[39m, encoding\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, errors\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m):\n\u001b[0;32m---> 18\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mloads\u001b[49m\u001b[43m(\u001b[49m\u001b[43mPath\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mread_text\u001b[49m\u001b[43m(\u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoding\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43merrors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43merrors\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n",
- "File \u001b[0;32m~/miniconda3/envs/nbdev/lib/python3.10/json/__init__.py:346\u001b[0m, in \u001b[0;36mloads\u001b[0;34m(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)\u001b[0m\n\u001b[1;32m 341\u001b[0m s \u001b[38;5;241m=\u001b[39m s\u001b[38;5;241m.\u001b[39mdecode(detect_encoding(s), \u001b[38;5;124m'\u001b[39m\u001b[38;5;124msurrogatepass\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 343\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m 344\u001b[0m parse_int \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m parse_float \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m\n\u001b[1;32m 345\u001b[0m parse_constant \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m object_pairs_hook \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m kw):\n\u001b[0;32m--> 346\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_default_decoder\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdecode\u001b[49m\u001b[43m(\u001b[49m\u001b[43ms\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 347\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m 348\u001b[0m \u001b[38;5;28mcls\u001b[39m \u001b[38;5;241m=\u001b[39m JSONDecoder\n",
- "File \u001b[0;32m~/miniconda3/envs/nbdev/lib/python3.10/json/decoder.py:340\u001b[0m, in \u001b[0;36mJSONDecoder.decode\u001b[0;34m(self, s, _w)\u001b[0m\n\u001b[1;32m 338\u001b[0m end \u001b[38;5;241m=\u001b[39m _w(s, end)\u001b[38;5;241m.\u001b[39mend()\n\u001b[1;32m 339\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m end \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mlen\u001b[39m(s):\n\u001b[0;32m--> 340\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m JSONDecodeError(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mExtra data\u001b[39m\u001b[38;5;124m\"\u001b[39m, s, end)\n\u001b[1;32m 341\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m obj\n",
- "\u001b[0;31mJSONDecodeError\u001b[0m: Extra data: line 1 column 3 (char 2)"
- ]
- }
- ],
+ "outputs": [],
"source": [
"#|eval: false\n",
"#|hide\n",
diff --git a/nbs/api/11_clean.ipynb b/nbs/api/11_clean.ipynb
index 3d983a266..7bc246891 100644
--- a/nbs/api/11_clean.ipynb
+++ b/nbs/api/11_clean.ipynb
@@ -399,7 +399,7 @@
" _write = partial(process_write, warn_msg='Failed to clean notebook', proc_nb=_clean)\n",
" if stdin: return _write(f_in=sys.stdin, f_out=sys.stdout)\n",
" if fname is None: fname = get_config().nbs_path\n",
- " for f in globtastic(fname, file_re=r'.*\\.ipynb$|.*\\.qmd$', skip_folder_re='^[_.]'): _write(f_in=f, disp=disp)"
+ " for f in globtastic(fname, file_re=r'.*\\.ipynb$', skip_folder_re='^[_.]'): _write(f_in=f, disp=disp) # Don't clean .qmd files"
]
},
{
From f113f14049c2fc7083af5cde6a2eac248ec229a8 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Wed, 28 May 2025 18:04:01 -0400
Subject: [PATCH 04/31] Simplify `read_qmd` in style of nbdev code
---
nbdev/process.py | 109 +++++-----------------------
nbs/api/03_process.ipynb | 148 +++++----------------------------------
2 files changed, 34 insertions(+), 223 deletions(-)
diff --git a/nbdev/process.py b/nbdev/process.py
index 205f71b58..15ba68ca7 100644
--- a/nbdev/process.py
+++ b/nbdev/process.py
@@ -18,107 +18,30 @@
from collections import defaultdict
# %% ../nbs/api/03_process.ipynb
-import re
-import json
-from pathlib import Path
-from fastcore.utils import AttrDict
-
-# Assuming AttrDict is available from fastcore.utils or similar
-# If not, a regular dictionary can be used for the structure
-# from fastcore.utils import AttrDict
-
def read_qmd(path):
"""Return notebook-like structure from a .qmd file"""
- path = Path(path)
- content = path.read_text(encoding='utf-8')
-
+ content = Path(path).read_text(encoding='utf-8')
+ # Code block markers split the .qmd content into cells
+ parts = re.split(r'^```\{[^}]*\}\s*$|^```\s*$', content, flags=re.MULTILINE)
cells = []
- current_cell_source = []
- current_cell_type = 'markdown' # Start assuming markdown
-
- # Regex to find fenced code blocks with language identifier
- code_block_start_re = re.compile(r'^```{\s*(\w+)\s*}')
- code_block_end_re = re.compile(r'^```\s*$')
-
- in_code_block = False
-
- # Iterate through lines to identify markdown and code cells
- for line in content.splitlines(keepends=True):
- code_match = code_block_start_re.match(line)
- code_end_match = code_block_end_re.match(line)
-
- if code_match:
- # If we were in a markdown cell, save it first
- if current_cell_source and current_cell_type == 'markdown':
- cells.append({
- 'cell_type': 'markdown',
- 'metadata': {},
- 'source': ''.join(current_cell_source)
- })
- current_cell_source = []
-
- # Start a new code cell
- current_cell_type = 'code'
- in_code_block = True
- # We skip the ````{language}` line itself in the cell source
-
- elif code_end_match and in_code_block:
- # End of a code block, save the code cell
- if current_cell_source:
- # Remove the last line if it's just the closing fence ```
- # (splitlines(keepends=True) includes the newline)
- if current_cell_source[-1].strip() == '```':
- current_cell_source.pop()
- cells.append({
- 'cell_type': 'code',
- 'execution_count': None, # .qmd files don't store execution count
- 'metadata': {}, # Metadata might be inferred later or added default
- 'outputs': [], # .qmd files don't store outputs
- 'source': ''.join(current_cell_source)
- })
- current_cell_source = []
- current_cell_type = 'markdown' # Next content is markdown
- in_code_block = False
-
- else:
- # Add line to the current cell source
- current_cell_source.append(line)
-
- # Add any remaining markdown content as a cell
- if current_cell_source and current_cell_type == 'markdown':
- cells.append({
- 'cell_type': 'markdown',
- 'metadata': {},
- 'source': ''.join(current_cell_source)
- })
-
- # Construct the final notebook-like dictionary
- # Use default values for nbformat and metadata that mimic ipynb
- nb_obj = {
+ for i, part in enumerate(parts):
+ if not part.strip(): continue
+ # Odd indices are code cells (between opening and closing ```)
+ cell_type = 'code' if i % 2 == 1 else 'markdown'
+ cell = { 'cell_type': cell_type, 'metadata': {}, 'source': part }
+ if cell_type == 'code': cell.update({'execution_count': None, 'outputs': []})
+ cells.append(cell)
+
+ return dict2nb({
'cells': cells,
'metadata': {
- 'kernelspec': {
- 'display_name': 'Python 3', # Default, could try to infer from ```{lang}
- 'language': 'python', # Default
- 'name': 'python3' # Default
- },
- 'language_info': { # Basic default language info
- 'name': 'python'
- }
+ 'kernelspec': {'display_name': 'Python 3', 'language': 'python', 'name': 'python3'},
+ 'language_info': {'name': 'python'}
},
'nbformat': 4,
- 'nbformat_minor': 5, # Use a recent minor version
+ 'nbformat_minor': 5,
'path_': str(path)
- }
-
- # If AttrDict is desired, convert here:
- return dict2nb(nb_obj)
- # return nb_obj
-
-# Example Usage (assuming minimal.qmd is in a 'tests' directory relative to execution)
-# qmd_path = 'tests/minimal.qmd'
-# nb_like_object = read_qmd(qmd_path)
-# print(json.dumps(nb_like_object, indent=2)) # Print to see the structure
+ })
# %% ../nbs/api/03_process.ipynb
# from https://github.com/quarto-dev/quarto-cli/blob/main/src/resources/jupyter/notebook.py
diff --git a/nbs/api/03_process.ipynb b/nbs/api/03_process.ipynb
index a7ed8fd2e..54fedcb50 100644
--- a/nbs/api/03_process.ipynb
+++ b/nbs/api/03_process.ipynb
@@ -79,108 +79,31 @@
"metadata": {},
"outputs": [],
"source": [
- "#| export\n",
- "import re\n",
- "import json\n",
- "from pathlib import Path\n",
- "from fastcore.utils import AttrDict\n",
- "\n",
- "# Assuming AttrDict is available from fastcore.utils or similar\n",
- "# If not, a regular dictionary can be used for the structure\n",
- "# from fastcore.utils import AttrDict\n",
- "\n",
+ "#|export\n",
"def read_qmd(path):\n",
" \"\"\"Return notebook-like structure from a .qmd file\"\"\"\n",
- " path = Path(path)\n",
- " content = path.read_text(encoding='utf-8')\n",
- "\n",
+ " content = Path(path).read_text(encoding='utf-8')\n",
+ " # Code block markers split the .qmd content into cells\n",
+ " parts = re.split(r'^```\\{[^}]*\\}\\s*$|^```\\s*$', content, flags=re.MULTILINE)\n",
" cells = []\n",
- " current_cell_source = []\n",
- " current_cell_type = 'markdown' # Start assuming markdown\n",
- "\n",
- " # Regex to find fenced code blocks with language identifier\n",
- " code_block_start_re = re.compile(r'^```{\\s*(\\w+)\\s*}')\n",
- " code_block_end_re = re.compile(r'^```\\s*$')\n",
- "\n",
- " in_code_block = False\n",
- "\n",
- " # Iterate through lines to identify markdown and code cells\n",
- " for line in content.splitlines(keepends=True):\n",
- " code_match = code_block_start_re.match(line)\n",
- " code_end_match = code_block_end_re.match(line)\n",
- "\n",
- " if code_match:\n",
- " # If we were in a markdown cell, save it first\n",
- " if current_cell_source and current_cell_type == 'markdown':\n",
- " cells.append({\n",
- " 'cell_type': 'markdown',\n",
- " 'metadata': {},\n",
- " 'source': ''.join(current_cell_source)\n",
- " })\n",
- " current_cell_source = []\n",
- "\n",
- " # Start a new code cell\n",
- " current_cell_type = 'code'\n",
- " in_code_block = True\n",
- " # We skip the ````{language}` line itself in the cell source\n",
- "\n",
- " elif code_end_match and in_code_block:\n",
- " # End of a code block, save the code cell\n",
- " if current_cell_source:\n",
- " # Remove the last line if it's just the closing fence ```\n",
- " # (splitlines(keepends=True) includes the newline)\n",
- " if current_cell_source[-1].strip() == '```':\n",
- " current_cell_source.pop()\n",
- " cells.append({\n",
- " 'cell_type': 'code',\n",
- " 'execution_count': None, # .qmd files don't store execution count\n",
- " 'metadata': {}, # Metadata might be inferred later or added default\n",
- " 'outputs': [], # .qmd files don't store outputs\n",
- " 'source': ''.join(current_cell_source)\n",
- " })\n",
- " current_cell_source = []\n",
- " current_cell_type = 'markdown' # Next content is markdown\n",
- " in_code_block = False\n",
- "\n",
- " else:\n",
- " # Add line to the current cell source\n",
- " current_cell_source.append(line)\n",
- "\n",
- " # Add any remaining markdown content as a cell\n",
- " if current_cell_source and current_cell_type == 'markdown':\n",
- " cells.append({\n",
- " 'cell_type': 'markdown',\n",
- " 'metadata': {},\n",
- " 'source': ''.join(current_cell_source)\n",
- " })\n",
- "\n",
- " # Construct the final notebook-like dictionary\n",
- " # Use default values for nbformat and metadata that mimic ipynb\n",
- " nb_obj = {\n",
+ " for i, part in enumerate(parts):\n",
+ " if not part.strip(): continue\n",
+ " # Odd indices are code cells (between opening and closing ```)\n",
+ " cell_type = 'code' if i % 2 == 1 else 'markdown'\n",
+ " cell = { 'cell_type': cell_type, 'metadata': {}, 'source': part }\n",
+ " if cell_type == 'code': cell.update({'execution_count': None, 'outputs': []})\n",
+ " cells.append(cell)\n",
+ "\n",
+ " return dict2nb({\n",
" 'cells': cells,\n",
" 'metadata': {\n",
- " 'kernelspec': {\n",
- " 'display_name': 'Python 3', # Default, could try to infer from ```{lang}\n",
- " 'language': 'python', # Default\n",
- " 'name': 'python3' # Default\n",
- " },\n",
- " 'language_info': { # Basic default language info\n",
- " 'name': 'python'\n",
- " }\n",
+ " 'kernelspec': {'display_name': 'Python 3', 'language': 'python', 'name': 'python3'},\n",
+ " 'language_info': {'name': 'python'}\n",
" },\n",
" 'nbformat': 4,\n",
- " 'nbformat_minor': 5, # Use a recent minor version\n",
+ " 'nbformat_minor': 5,\n",
" 'path_': str(path)\n",
- " }\n",
- "\n",
- " # If AttrDict is desired, convert here:\n",
- " return dict2nb(nb_obj)\n",
- " # return nb_obj\n",
- "\n",
- "# Example Usage (assuming minimal.qmd is in a 'tests' directory relative to execution)\n",
- "# qmd_path = 'tests/minimal.qmd'\n",
- "# nb_like_object = read_qmd(qmd_path)\n",
- "# print(json.dumps(nb_like_object, indent=2)) # Print to see the structure"
+ " })"
]
},
{
@@ -578,48 +501,13 @@
"execution_count": null,
"id": "1645c63d",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "---\n",
- "title: Foo\n",
- "execute:\n",
- " echo: false\n",
- "---\n",
- "\n",
- "# A Title\n",
- "\n",
- "> A description\n",
- "\n",
- "This notebook is used to demonstrate and test all the features of nbdev's export functionality. See the notebooks in `nbs` for how it's used.\n",
- "\n",
- "\n",
- "\n",
- "exec(\"o_y=1\")\n",
- "exec(\"p_y=1\")\n",
- "_all_ = [o_y, 'p_y']\n",
- "\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"everything_fn_qmd = '../../tests/01_everything.qmd'\n",
"\n",
"NBProcessor(everything_fn_qmd, print_execs).process()"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "d4772302",
- "metadata": {},
- "outputs": [],
- "source": [
- "NBProcessor(everything_fn_qmd, print_execs).process()"
- ]
- },
{
"cell_type": "markdown",
"id": "a8202589",
From 3a8d61b7f5f037970ce1663b389a3b6d0852a9b0 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Thu, 29 May 2025 12:32:10 -0400
Subject: [PATCH 05/31] Allow custom directives in `.qmd` files
---
nbdev/serve.py | 2 +-
nbs/api/17_serve.ipynb | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/nbdev/serve.py b/nbdev/serve.py
index d7d0447df..c9fce81e2 100644
--- a/nbdev/serve.py
+++ b/nbdev/serve.py
@@ -46,7 +46,7 @@ def _proc_file(s, cache, path, mtime=None):
if s.stat().st_mtime<=dtime: return
d.parent.mkdir(parents=True, exist_ok=True)
- if s.suffix=='.ipynb': return s,d,FilterDefaults
+ if s.suffix in ['.ipynb','.qmd']: return s,d,FilterDefaults
md = _is_qpy(s)
if md is not None: return s,d,md.strip()
else: copy2(s,d)
diff --git a/nbs/api/17_serve.ipynb b/nbs/api/17_serve.ipynb
index 25bdfa0de..8198c6dc1 100644
--- a/nbs/api/17_serve.ipynb
+++ b/nbs/api/17_serve.ipynb
@@ -96,7 +96,7 @@
" if s.stat().st_mtime<=dtime: return\n",
"\n",
" d.parent.mkdir(parents=True, exist_ok=True)\n",
- " if s.suffix=='.ipynb': return s,d,FilterDefaults\n",
+ " if s.suffix in ['.ipynb','.qmd']: return s,d,FilterDefaults\n",
" md = _is_qpy(s)\n",
" if md is not None: return s,d,md.strip()\n",
" else: copy2(s,d)"
From c7389eb7ab4bcc024127ac8f88ee782cdc8755a5 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Thu, 29 May 2025 12:37:29 -0400
Subject: [PATCH 06/31] Attempt .qmd to generate good looking docs
---
index.ipynb | 140 +++++++++++
index_ipynb.ipynb | 112 +++++++++
nbdev/_modidx.py | 3 +-
nbdev/process.py | 79 ++++--
nbdev/quarto.py | 5 +-
nbdev/serve_drv.py | 10 +-
nbs/api/03_process.ipynb | 528 +++++++++++++++++++++++++++++++++++++--
nbs/api/14_quarto.ipynb | 5 +-
tests/tst_index.ipynb | 112 +++++++++
tests/tst_index.qmd | 74 ++++++
tst_index_pandoc.ipynb | 82 ++++++
11 files changed, 1101 insertions(+), 49 deletions(-)
create mode 100644 index.ipynb
create mode 100644 index_ipynb.ipynb
create mode 100644 tests/tst_index.ipynb
create mode 100644 tests/tst_index.qmd
create mode 100644 tst_index_pandoc.ipynb
diff --git a/index.ipynb b/index.ipynb
new file mode 100644
index 000000000..7ea983e44
--- /dev/null
+++ b/index.ipynb
@@ -0,0 +1,140 @@
+{
+ "cells": [
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "---\n",
+ "output-file: index.html\n",
+ "title: HAMUX\n",
+ "\n",
+ "---\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ ""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "language": "python"
+ },
+ "outputs": [],
+ "source": [
+ "#| echo: false\n",
+ "# from hamux_qmd.core import *"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "> Energy formulation for deep learning\n",
+ "\n",
+ "This file will become your README and also the index of your documentation.\n",
+ "\n",
+ "## Developer Guide\n",
+ "\n",
+ "If you are new to using `nbdev` here are some useful pointers to get you started.\n",
+ "\n",
+ "### Install hamux_qmd in Development mode\n",
+ "\n",
+ "```sh\n",
+ "# make sure hamux_qmd package is installed in development mode\n",
+ "$ pip install -e .\n",
+ "\n",
+ "# make changes under nbs/ directory\n",
+ "# ...\n",
+ "\n",
+ "# compile to have changes apply to hamux_qmd\n",
+ "$ nbdev_prepare"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "language": "python"
+ },
+ "outputs": [],
+ "source": [
+ "## Usage\n",
+ "\n",
+ "### Installation\n",
+ "\n",
+ "Install latest from the GitHub [repository][repo]:\n",
+ "\n",
+ "```sh\n",
+ "$ pip install git+https://github.com/bhoov/hamux_qmd.git"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "or from [conda][conda]\n",
+ "\n",
+ "```sh\n",
+ "$ conda install -c bhoov hamux_qmd"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "language": "python"
+ },
+ "outputs": [],
+ "source": [
+ "or from [pypi][pypi]\n",
+ "\n",
+ "\n",
+ "```sh\n",
+ "$ pip install hamux_qmd"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "[repo]: https://github.com/bhoov/hamux_qmd\n",
+ "[docs]: https://bhoov.github.io/hamux_qmd/\n",
+ "[pypi]: https://pypi.org/project/hamux_qmd/\n",
+ "[conda]: https://anaconda.org/bhoov/hamux_qmd\n",
+ "\n",
+ "## How to use\n",
+ "\n",
+ "Fill me in please! Don't forget code examples:\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "language": "python"
+ },
+ "outputs": [],
+ "source": [
+ "1+1"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/index_ipynb.ipynb b/index_ipynb.ipynb
new file mode 100644
index 000000000..ba620d042
--- /dev/null
+++ b/index_ipynb.ipynb
@@ -0,0 +1,112 @@
+{
+ "cells": [
+ {
+ "cell_type": "raw",
+ "metadata": {},
+ "source": [
+ "---\n",
+ "output-file: index_ipynb.html\n",
+ "title: HAMUX2\n",
+ "\n",
+ "---\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ ""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "language": "python"
+ },
+ "outputs": [],
+ "source": [
+ "#| echo: false\n",
+ "# from hamux_qmd.core import *"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "> Energy formulation for deep learning\n",
+ "\n",
+ "This file will become your README and also the index of your documentation.\n",
+ "\n",
+ "## Developer Guide\n",
+ "\n",
+ "If you are new to using `nbdev` here are some useful pointers to get you started.\n",
+ "\n",
+ "### Install hamux_qmd in Development mode\n",
+ "\n",
+ "```sh\n",
+ "# make sure hamux_qmd package is installed in development mode\n",
+ "$ pip install -e .\n",
+ "\n",
+ "# make changes under nbs/ directory\n",
+ "# ...\n",
+ "\n",
+ "# compile to have changes apply to hamux_qmd\n",
+ "$ nbdev_prepare\n",
+ "```\n",
+ "\n",
+ "## Usage\n",
+ "\n",
+ "### Installation\n",
+ "\n",
+ "Install latest from the GitHub [repository][repo]:\n",
+ "\n",
+ "```sh\n",
+ "$ pip install git+https://github.com/bhoov/hamux_qmd.git\n",
+ "```\n",
+ "\n",
+ "or from [conda][conda]\n",
+ "\n",
+ "```sh\n",
+ "$ conda install -c bhoov hamux_qmd\n",
+ "```\n",
+ "\n",
+ "or from [pypi][pypi]\n",
+ "\n",
+ "\n",
+ "```sh\n",
+ "$ pip install hamux_qmd\n",
+ "```\n",
+ "\n",
+ "\n",
+ "[repo]: https://github.com/bhoov/hamux_qmd\n",
+ "[docs]: https://bhoov.github.io/hamux_qmd/\n",
+ "[pypi]: https://pypi.org/project/hamux_qmd/\n",
+ "[conda]: https://anaconda.org/bhoov/hamux_qmd\n",
+ "\n",
+ "## How to use\n",
+ "\n",
+ "Fill me in please! Don't forget code examples:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "language": "python"
+ },
+ "outputs": [],
+ "source": [
+ "1+1"
+ ]
+ }
+ ],
+ "metadata": {
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py
index caf820192..f1c2f3fb0 100644
--- a/nbdev/_modidx.py
+++ b/nbdev/_modidx.py
@@ -177,13 +177,14 @@
'nbdev.process._mk_procs': ('api/process.html#_mk_procs', 'nbdev/process.py'),
'nbdev.process._norm_quarto': ('api/process.html#_norm_quarto', 'nbdev/process.py'),
'nbdev.process._partition_cell': ('api/process.html#_partition_cell', 'nbdev/process.py'),
+ 'nbdev.process._qmd_to_raw_cell': ('api/process.html#_qmd_to_raw_cell', 'nbdev/process.py'),
'nbdev.process._quarto_re': ('api/process.html#_quarto_re', 'nbdev/process.py'),
- 'nbdev.process._read_nb_or_qmd': ('api/process.html#_read_nb_or_qmd', 'nbdev/process.py'),
'nbdev.process.extract_directives': ('api/process.html#extract_directives', 'nbdev/process.py'),
'nbdev.process.first_code_ln': ('api/process.html#first_code_ln', 'nbdev/process.py'),
'nbdev.process.instantiate': ('api/process.html#instantiate', 'nbdev/process.py'),
'nbdev.process.nb_lang': ('api/process.html#nb_lang', 'nbdev/process.py'),
'nbdev.process.opt_set': ('api/process.html#opt_set', 'nbdev/process.py'),
+ 'nbdev.process.read_nb_or_qmd': ('api/process.html#read_nb_or_qmd', 'nbdev/process.py'),
'nbdev.process.read_qmd': ('api/process.html#read_qmd', 'nbdev/process.py')},
'nbdev.processors': { 'nbdev.processors.FilterDefaults': ('api/processors.html#filterdefaults', 'nbdev/processors.py'),
'nbdev.processors.FilterDefaults.__call__': ( 'api/processors.html#filterdefaults.__call__',
diff --git a/nbdev/process.py b/nbdev/process.py
index 15ba68ca7..44bb19716 100644
--- a/nbdev/process.py
+++ b/nbdev/process.py
@@ -3,8 +3,8 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/03_process.ipynb.
# %% auto 0
-__all__ = ['langs', 'read_qmd', 'nb_lang', 'first_code_ln', 'extract_directives', 'opt_set', 'instantiate', 'NBProcessor',
- 'Processor']
+__all__ = ['langs', 'read_qmd', 'nb_lang', 'first_code_ln', 'extract_directives', 'opt_set', 'instantiate', 'read_nb_or_qmd',
+ 'NBProcessor', 'Processor']
# %% ../nbs/api/03_process.ipynb
from .config import *
@@ -18,30 +18,67 @@
from collections import defaultdict
# %% ../nbs/api/03_process.ipynb
-def read_qmd(path):
- """Return notebook-like structure from a .qmd file"""
+def _qmd_to_raw_cell(source_str, cell_type_str):
+ """Helper to create a raw cell dictionary (like from a .ipynb JSON)."""
+ # Source is typically a list of strings in .ipynb, but a single string is also common.
+ # NbCell's set_source method handles both by doing ''.join(source).
+ # Here, we'll provide it as a single string, which is what re.split gives us.
+ cell = {
+ 'cell_type': cell_type_str,
+ 'metadata': {}, # Standard empty metadata
+ 'source': source_str # Already stripped by the caller
+ }
+ if cell_type_str == 'code':
+ cell['execution_count'] = None # In JSON, this is null
+ cell['outputs'] = []
+ return cell
+
+def read_qmd(path): # Renamed to clarify output
+ """
+ Reads a .qmd file and returns a notebook structure as a JSON-like dictionary,
+ compatible with execnb.nbio.dict2nb.
+ """
content = Path(path).read_text(encoding='utf-8')
- # Code block markers split the .qmd content into cells
- parts = re.split(r'^```\{[^}]*\}\s*$|^```\s*$', content, flags=re.MULTILINE)
- cells = []
- for i, part in enumerate(parts):
- if not part.strip(): continue
- # Odd indices are code cells (between opening and closing ```)
- cell_type = 'code' if i % 2 == 1 else 'markdown'
- cell = { 'cell_type': cell_type, 'metadata': {}, 'source': part }
- if cell_type == 'code': cell.update({'execution_count': None, 'outputs': []})
- cells.append(cell)
-
- return dict2nb({
- 'cells': cells,
+ cell_pat = re.compile(r"^(`{3,})\s*\{python[^\n]*\}\s*(.*?)^\1\s*$", re.MULTILINE | re.DOTALL)
+
+ # parts will be [md_chunk, captured_backticks_1, captured_code_1, md_chunk_2, ...]
+ parts = cell_pat.split(content)
+ raw_cells = []
+
+ # Handle the first markdown segment (before any code cells, or all content if no code cells)
+ initial_md_source = parts[0].strip()
+ if initial_md_source: raw_cells.append(_qmd_to_raw_cell(initial_md_source, 'markdown'))
+
+ # Process subsequent parts: captured code (parts[i+1]) and following markdown (parts[i+2])
+ # Loop from the first captured group (backticks, at index 1) with a step of 3.
+ for i in range(1, len(parts), 3):
+ # Code cell content is in parts[i+1] (second capture group)
+ if i + 1 < len(parts):
+ code_source = parts[i+1].strip()
+ # Only add non-empty code cells, though regex usually ensures content.
+ if code_source:
+ raw_cells.append(_qmd_to_raw_cell(code_source, 'code'))
+
+ # Markdown cell content following the code cell is in parts[i+2]
+ if i + 2 < len(parts):
+ intermediate_md_source = parts[i+2].strip()
+ if intermediate_md_source:
+ raw_cells.append(_qmd_to_raw_cell(intermediate_md_source, 'markdown'))
+
+ # Construct the final notebook dictionary
+ notebook_dict = {
+ 'cells': raw_cells,
'metadata': {
'kernelspec': {'display_name': 'Python 3', 'language': 'python', 'name': 'python3'},
- 'language_info': {'name': 'python'}
+ 'language_info': {'name': 'python'},
+ 'path': str(path)
},
'nbformat': 4,
'nbformat_minor': 5,
'path_': str(path)
- })
+ }
+
+ return dict2nb(notebook_dict)
# %% ../nbs/api/03_process.ipynb
# from https://github.com/quarto-dev/quarto-cli/blob/main/src/resources/jupyter/notebook.py
@@ -116,7 +153,7 @@ def _mk_procs(procs, nb): return L(procs).map(instantiate, nb=nb)
def _is_direc(f): return getattr(f, '__name__', '-')[-1]=='_'
# %% ../nbs/api/03_process.ipynb
-def _read_nb_or_qmd(path):
+def read_nb_or_qmd(path):
if Path(path).suffix == '.qmd': return read_qmd(path)
return read_nb(path)
@@ -124,7 +161,7 @@ def _read_nb_or_qmd(path):
class NBProcessor:
"Process cells and nbdev comments in a notebook"
def __init__(self, path=None, procs=None, nb=None, debug=False, rm_directives=True, process=False):
- self.nb = _read_nb_or_qmd(path) if nb is None else nb
+ self.nb = read_nb_or_qmd(path) if nb is None else nb
self.lang = nb_lang(self.nb)
for cell in self.nb.cells: cell.directives_ = extract_directives(cell, remove=rm_directives, lang=self.lang)
self.procs = _mk_procs(procs, nb=self.nb)
diff --git a/nbdev/quarto.py b/nbdev/quarto.py
index 4742d1c6e..28a5448d7 100644
--- a/nbdev/quarto.py
+++ b/nbdev/quarto.py
@@ -149,6 +149,7 @@ def _f(a,b): return Path(a),b
css: styles.css
toc: true
keep-md: true
+ keep-ipynb: true
commonmark: default
website:
@@ -205,6 +206,7 @@ def _pre_docs(path=None, n_workers:int=defaults.cpus, **kwargs):
nbdev.doclinks._build_modidx()
nbdev_sidebar.__wrapped__(path=path, **kwargs)
cache = proc_nbs(path, n_workers=n_workers, **kwargs)
+ print("cache: ", cache)
return cache,cfg,path
# %% ../nbs/api/14_quarto.ipynb
@@ -264,7 +266,8 @@ def nbdev_readme(
with _SidebarYmlRemoved(path): # to avoid rendering whole website
cache = proc_nbs(path)
- _sprun(f'cd "{cache}" && quarto render "{cache/cfg.readme_nb}" -o README.md -t gfm --no-execute')
+ readme_nb_name = Path(cfg.readme_nb).with_suffix('.ipynb')
+ _sprun(f'cd "{cache}" && quarto render "{cache/readme_nb_name}" -o README.md -t gfm --no-execute')
_save_cached_readme(cache, cfg)
diff --git a/nbdev/serve_drv.py b/nbdev/serve_drv.py
index bfd42b500..a20a5a1aa 100644
--- a/nbdev/serve_drv.py
+++ b/nbdev/serve_drv.py
@@ -1,5 +1,6 @@
import os
-from execnb.nbio import read_nb,write_nb
+from execnb.nbio import write_nb
+from nbdev.process import read_nb_or_qmd
from io import StringIO
from contextlib import redirect_stdout
@@ -12,14 +13,17 @@ def exec_scr(src, dst, md):
dst.write_text(res + f.getvalue())
def exec_nb(src, dst, cb):
- nb = read_nb(src)
+ print("src", src)
+ if src.suffix == ".qmd": dst = dst.with_suffix(".ipynb")
+ print("dst", dst)
+ nb = read_nb_or_qmd(src)
cb()(nb)
write_nb(nb, dst)
def main(o):
src,dst,x = o
os.environ["IN_TEST"] = "1"
- if src.suffix=='.ipynb': exec_nb(src, dst, x)
+ if src.suffix in ['.ipynb','.qmd']: exec_nb(src, dst, x)
elif src.suffix=='.py': exec_scr(src, dst, x)
else: raise Exception(src)
del os.environ["IN_TEST"]
diff --git a/nbs/api/03_process.ipynb b/nbs/api/03_process.ipynb
index 54fedcb50..fc4f703b8 100644
--- a/nbs/api/03_process.ipynb
+++ b/nbs/api/03_process.ipynb
@@ -80,30 +80,484 @@
"outputs": [],
"source": [
"#|export\n",
- "def read_qmd(path):\n",
- " \"\"\"Return notebook-like structure from a .qmd file\"\"\"\n",
- " content = Path(path).read_text(encoding='utf-8')\n",
- " # Code block markers split the .qmd content into cells\n",
- " parts = re.split(r'^```\\{[^}]*\\}\\s*$|^```\\s*$', content, flags=re.MULTILINE)\n",
- " cells = []\n",
- " for i, part in enumerate(parts):\n",
- " if not part.strip(): continue\n",
- " # Odd indices are code cells (between opening and closing ```)\n",
- " cell_type = 'code' if i % 2 == 1 else 'markdown'\n",
- " cell = { 'cell_type': cell_type, 'metadata': {}, 'source': part }\n",
- " if cell_type == 'code': cell.update({'execution_count': None, 'outputs': []})\n",
- " cells.append(cell)\n",
+ "def _qmd_to_raw_cell(source_str, cell_type_str):\n",
+ " \"\"\"Helper to create a raw cell dictionary (like from a .ipynb JSON).\"\"\"\n",
+ " # Source is typically a list of strings in .ipynb, but a single string is also common.\n",
+ " # NbCell's set_source method handles both by doing ''.join(source).\n",
+ " # Here, we'll provide it as a single string, which is what re.split gives us.\n",
+ " cell = {\n",
+ " 'cell_type': cell_type_str,\n",
+ " 'metadata': {}, # Standard empty metadata\n",
+ " 'source': source_str # Already stripped by the caller\n",
+ " }\n",
+ " if cell_type_str == 'code':\n",
+ " cell['execution_count'] = None # In JSON, this is null\n",
+ " cell['outputs'] = []\n",
+ " return cell\n",
"\n",
- " return dict2nb({\n",
- " 'cells': cells,\n",
+ "def read_qmd(path): # Renamed to clarify output\n",
+ " \"\"\"\n",
+ " Reads a .qmd file and returns a notebook structure as a JSON-like dictionary,\n",
+ " compatible with execnb.nbio.dict2nb.\n",
+ " \"\"\"\n",
+ " content = Path(path).read_text(encoding='utf-8')\n",
+ " cell_pat = re.compile(r\"^(`{3,})\\s*\\{python[^\\n]*\\}\\s*(.*?)^\\1\\s*$\", re.MULTILINE | re.DOTALL)\n",
+ " \n",
+ " # parts will be [md_chunk, captured_backticks_1, captured_code_1, md_chunk_2, ...]\n",
+ " parts = cell_pat.split(content)\n",
+ " raw_cells = []\n",
+ " \n",
+ " # Handle the first markdown segment (before any code cells, or all content if no code cells)\n",
+ " initial_md_source = parts[0].strip()\n",
+ " if initial_md_source: raw_cells.append(_qmd_to_raw_cell(initial_md_source, 'markdown'))\n",
+ " \n",
+ " # Process subsequent parts: captured code (parts[i+1]) and following markdown (parts[i+2])\n",
+ " # Loop from the first captured group (backticks, at index 1) with a step of 3.\n",
+ " for i in range(1, len(parts), 3):\n",
+ " # Code cell content is in parts[i+1] (second capture group)\n",
+ " if i + 1 < len(parts):\n",
+ " code_source = parts[i+1].strip()\n",
+ " # Only add non-empty code cells, though regex usually ensures content.\n",
+ " if code_source: \n",
+ " raw_cells.append(_qmd_to_raw_cell(code_source, 'code'))\n",
+ " \n",
+ " # Markdown cell content following the code cell is in parts[i+2]\n",
+ " if i + 2 < len(parts):\n",
+ " intermediate_md_source = parts[i+2].strip()\n",
+ " if intermediate_md_source:\n",
+ " raw_cells.append(_qmd_to_raw_cell(intermediate_md_source, 'markdown'))\n",
+ " \n",
+ " # Construct the final notebook dictionary\n",
+ " notebook_dict = {\n",
+ " 'cells': raw_cells,\n",
" 'metadata': {\n",
" 'kernelspec': {'display_name': 'Python 3', 'language': 'python', 'name': 'python3'},\n",
- " 'language_info': {'name': 'python'}\n",
+ " 'language_info': {'name': 'python'},\n",
+ " 'path': str(path)\n",
" },\n",
" 'nbformat': 4,\n",
" 'nbformat_minor': 5,\n",
" 'path_': str(path)\n",
- " })"
+ " }\n",
+ " \n",
+ " return dict2nb(notebook_dict)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1559503a",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "ipynb: 5\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "[{'cell_type': 'markdown',\n",
+ " 'metadata': {},\n",
+ " 'source': '---\\ntitle: \"HAMUX QMD\"\\n---\\n\\n# HAMUX QMD\\n> Energy formulation for deep learning',\n",
+ " 'idx_': 0,\n",
+ " 'directives_': {}},\n",
+ " {'cell_type': 'code',\n",
+ " 'metadata': {},\n",
+ " 'source': '#| echo: false\\n# from hamux_qmd.core import *',\n",
+ " 'execution_count': None,\n",
+ " 'outputs': [],\n",
+ " 'idx_': 1,\n",
+ " 'directives_': {'echo:': ['false']}},\n",
+ " {'cell_type': 'code',\n",
+ " 'metadata': {},\n",
+ " 'source': '#| echo: true \\nprint(3+4)',\n",
+ " 'execution_count': None,\n",
+ " 'outputs': [],\n",
+ " 'idx_': 2,\n",
+ " 'directives_': {'echo:': ['true']}},\n",
+ " {'cell_type': 'markdown',\n",
+ " 'metadata': {},\n",
+ " 'source': \"This file will become your README and also the index of your documentation.\\n\\n## Developer Guide\\n\\nIf you are new to using `nbdev` here are some useful pointers to get you started.\\n\\n### Install hamux_qmd in Development mode\\n\\n```sh\\n# make sure hamux_qmd package is installed in development mode\\n$ pip install -e .\\n\\n# make changes under nbs/ directory\\n# ...\\n\\n# compile to have changes apply to hamux_qmd\\n$ nbdev_prepare\\n```\\n\\n## Usage\\n\\n### Installation\\n\\nInstall latest from the GitHub [repository][repo]:\\n\\n```sh\\n$ pip install git+https://github.com/bhoov/hamux_qmd.git\\n```\\n\\nor from [conda][conda]\\n\\n```sh\\n$ conda install -c bhoov hamux_qmd\\n```\\n\\nor from [pypi][pypi]\\n\\n\\n```sh\\n$ pip install hamux_qmd\\n```\\n\\n\\n[repo]: https://github.com/bhoov/hamux_qmd\\n[docs]: https://bhoov.github.io/hamux_qmd/\\n[pypi]: https://pypi.org/project/hamux_qmd/\\n[conda]: https://anaconda.org/bhoov/hamux_qmd\\n\\n## How to use\\n\\nFill me in please! Don't forget code examples:\",\n",
+ " 'idx_': 3,\n",
+ " 'directives_': {}},\n",
+ " {'cell_type': 'code',\n",
+ " 'metadata': {},\n",
+ " 'source': '1+1',\n",
+ " 'execution_count': None,\n",
+ " 'outputs': [],\n",
+ " 'idx_': 4,\n",
+ " 'directives_': {}}]"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Test: I want the .qmd file to be the exact same .ipynb file in the nb source (except for outputs)\n",
+ "from nbdev.process import NBProcessor\n",
+ "tst_ipynb = '../../tests/tst_index.ipynb'\n",
+ "tst_qmd = '../../tests/tst_index.qmd'\n",
+ "\n",
+ "nbproc_ipynb = NBProcessor(tst_ipynb)\n",
+ "nbproc_qmd = NBProcessor(tst_qmd)\n",
+ "nbproc_ipynb.nb.cells"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "d6c39f04",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "('```', '#| export\\nprint(3+4)\\n')\n",
+ "('````', '#| export\\nprint(9+12)\\n')\n",
+ "('`````', 'print(5+6)\\n')\n"
+ ]
+ }
+ ],
+ "source": [
+ "tst_cell = \"\"\"\\\n",
+ "# Title \n",
+ "> description\n",
+ "\n",
+ "and a couple more pieces of information before the code cell:\n",
+ "\n",
+ "``` {python .code-cell-2}\n",
+ "#| export\n",
+ "print(3+4)\n",
+ "```\n",
+ "\n",
+ "```` {python .code-cell-2}\n",
+ "#| export\n",
+ "print(9+12)\n",
+ "````\n",
+ "\n",
+ "`````{python}\n",
+ "print(5+6)\n",
+ "`````\n",
+ "\n",
+ "```python\n",
+ "print(\"not a cell\")\n",
+ "```\n",
+ "\"\"\"\n",
+ "cell_pat = re.compile(r\"^(`{3,})\\s*\\{python[^\\n]*\\}\\s*(.*?)^\\1\\s*$\", re.MULTILINE | re.DOTALL)\n",
+ "matches = cell_pat.findall(tst_cell)\n",
+ "for match in matches: print(match)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a315bb32",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/markdown": [
+ "```json\n",
+ "{ 'cells': [ { 'cell_type': 'markdown',\n",
+ " 'idx_': 0,\n",
+ " 'metadata': {},\n",
+ " 'source': '---\\n'\n",
+ " 'title: \"HAMUX QMD\"\\n'\n",
+ " '---\\n'\n",
+ " '\\n'\n",
+ " '# HAMUX QMD\\n'\n",
+ " '> Energy formulation for deep learning'},\n",
+ " { 'cell_type': 'code',\n",
+ " 'execution_count': None,\n",
+ " 'idx_': 1,\n",
+ " 'metadata': {},\n",
+ " 'outputs': [],\n",
+ " 'source': '#| echo: false\\n# from hamux_qmd.core import *'},\n",
+ " { 'cell_type': 'markdown',\n",
+ " 'idx_': 2,\n",
+ " 'metadata': {},\n",
+ " 'source': 'This is an insertion of another bit of markdown.'},\n",
+ " { 'cell_type': 'code',\n",
+ " 'execution_count': None,\n",
+ " 'idx_': 3,\n",
+ " 'metadata': {},\n",
+ " 'outputs': [],\n",
+ " 'source': '#| echo: true\\nprint(6+7)'},\n",
+ " { 'cell_type': 'markdown',\n",
+ " 'idx_': 4,\n",
+ " 'metadata': {},\n",
+ " 'source': 'This file will become your README and also the index '\n",
+ " 'of your documentation.\\n'\n",
+ " '\\n'\n",
+ " '## Developer Guide\\n'\n",
+ " '\\n'\n",
+ " 'If you are new to using `nbdev` here are some useful '\n",
+ " 'pointers to get you started.\\n'\n",
+ " '\\n'\n",
+ " '### Install hamux_qmd in Development mode\\n'\n",
+ " '\\n'\n",
+ " '```sh\\n'\n",
+ " '# make sure hamux_qmd package is installed in '\n",
+ " 'development mode\\n'\n",
+ " '$ pip install -e .\\n'\n",
+ " '\\n'\n",
+ " '# make changes under nbs/ directory\\n'\n",
+ " '# ...\\n'\n",
+ " '\\n'\n",
+ " '# compile to have changes apply to hamux_qmd\\n'\n",
+ " '$ nbdev_prepare\\n'\n",
+ " '```\\n'\n",
+ " '\\n'\n",
+ " '## Usage\\n'\n",
+ " '\\n'\n",
+ " '### Installation\\n'\n",
+ " '\\n'\n",
+ " 'Install latest from the GitHub [repository][repo]:\\n'\n",
+ " '\\n'\n",
+ " '```sh\\n'\n",
+ " '$ pip install '\n",
+ " 'git+https://github.com/bhoov/hamux_qmd.git\\n'\n",
+ " '```\\n'\n",
+ " '\\n'\n",
+ " 'or from [conda][conda]\\n'\n",
+ " '\\n'\n",
+ " '```sh\\n'\n",
+ " '$ conda install -c bhoov hamux_qmd\\n'\n",
+ " '```\\n'\n",
+ " '\\n'\n",
+ " 'or from [pypi][pypi]\\n'\n",
+ " '\\n'\n",
+ " '\\n'\n",
+ " '```sh\\n'\n",
+ " '$ pip install hamux_qmd\\n'\n",
+ " '```\\n'\n",
+ " '\\n'\n",
+ " '\\n'\n",
+ " '[repo]: https://github.com/bhoov/hamux_qmd\\n'\n",
+ " '[docs]: https://bhoov.github.io/hamux_qmd/\\n'\n",
+ " '[pypi]: https://pypi.org/project/hamux_qmd/\\n'\n",
+ " '[conda]: https://anaconda.org/bhoov/hamux_qmd\\n'\n",
+ " '\\n'\n",
+ " '## How to use\\n'\n",
+ " '\\n'\n",
+ " \"Fill me in please! Don't forget code examples:\"},\n",
+ " { 'cell_type': 'code',\n",
+ " 'execution_count': None,\n",
+ " 'idx_': 5,\n",
+ " 'metadata': {},\n",
+ " 'outputs': [],\n",
+ " 'source': '1+1'}],\n",
+ " 'metadata': { 'kernelspec': { 'display_name': 'Python 3',\n",
+ " 'language': 'python',\n",
+ " 'name': 'python3'},\n",
+ " 'language_info': {'name': 'python'},\n",
+ " 'path_': '../../tests/tst_index.qmd'},\n",
+ " 'nbformat': 4,\n",
+ " 'nbformat_minor': 5}\n",
+ "```"
+ ],
+ "text/plain": [
+ "{'cells': [{'cell_type': 'markdown',\n",
+ " 'metadata': {},\n",
+ " 'source': '---\\ntitle: \"HAMUX QMD\"\\n---\\n\\n# HAMUX QMD\\n> Energy formulation for deep learning',\n",
+ " 'idx_': 0},\n",
+ " {'cell_type': 'code',\n",
+ " 'metadata': {},\n",
+ " 'source': '#| echo: false\\n# from hamux_qmd.core import *',\n",
+ " 'execution_count': None,\n",
+ " 'outputs': [],\n",
+ " 'idx_': 1},\n",
+ " {'cell_type': 'markdown',\n",
+ " 'metadata': {},\n",
+ " 'source': 'This is an insertion of another bit of markdown.',\n",
+ " 'idx_': 2},\n",
+ " {'cell_type': 'code',\n",
+ " 'metadata': {},\n",
+ " 'source': '#| echo: true\\nprint(6+7)',\n",
+ " 'execution_count': None,\n",
+ " 'outputs': [],\n",
+ " 'idx_': 3},\n",
+ " {'cell_type': 'markdown',\n",
+ " 'metadata': {},\n",
+ " 'source': \"This file will become your README and also the index of your documentation.\\n\\n## Developer Guide\\n\\nIf you are new to using `nbdev` here are some useful pointers to get you started.\\n\\n### Install hamux_qmd in Development mode\\n\\n```sh\\n# make sure hamux_qmd package is installed in development mode\\n$ pip install -e .\\n\\n# make changes under nbs/ directory\\n# ...\\n\\n# compile to have changes apply to hamux_qmd\\n$ nbdev_prepare\\n```\\n\\n## Usage\\n\\n### Installation\\n\\nInstall latest from the GitHub [repository][repo]:\\n\\n```sh\\n$ pip install git+https://github.com/bhoov/hamux_qmd.git\\n```\\n\\nor from [conda][conda]\\n\\n```sh\\n$ conda install -c bhoov hamux_qmd\\n```\\n\\nor from [pypi][pypi]\\n\\n\\n```sh\\n$ pip install hamux_qmd\\n```\\n\\n\\n[repo]: https://github.com/bhoov/hamux_qmd\\n[docs]: https://bhoov.github.io/hamux_qmd/\\n[pypi]: https://pypi.org/project/hamux_qmd/\\n[conda]: https://anaconda.org/bhoov/hamux_qmd\\n\\n## How to use\\n\\nFill me in please! Don't forget code examples:\",\n",
+ " 'idx_': 4},\n",
+ " {'cell_type': 'code',\n",
+ " 'metadata': {},\n",
+ " 'source': '1+1',\n",
+ " 'execution_count': None,\n",
+ " 'outputs': [],\n",
+ " 'idx_': 5}],\n",
+ " 'metadata': {'kernelspec': {'display_name': 'Python 3',\n",
+ " 'language': 'python',\n",
+ " 'name': 'python3'},\n",
+ " 'language_info': {'name': 'python'},\n",
+ " 'path_': '../../tests/tst_index.qmd'},\n",
+ " 'nbformat': 4,\n",
+ " 'nbformat_minor': 5}"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "read_qmd(\"../../tests/tst_index.qmd\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "a9481494",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/markdown": [
+ "```json\n",
+ "{ 'cells': [ { 'cell_type': 'markdown',\n",
+ " 'idx_': 0,\n",
+ " 'metadata': {},\n",
+ " 'source': '---\\n'\n",
+ " 'title: \"HAMUX QMD\"\\n'\n",
+ " '---\\n'\n",
+ " '\\n'\n",
+ " '# HAMUX QMD\\n'\n",
+ " '> Energy formulation for deep learning'},\n",
+ " { 'cell_type': 'code',\n",
+ " 'execution_count': None,\n",
+ " 'idx_': 1,\n",
+ " 'metadata': {},\n",
+ " 'outputs': [],\n",
+ " 'source': '#| echo: false\\n# from hamux_qmd.core import *'},\n",
+ " { 'cell_type': 'code',\n",
+ " 'execution_count': None,\n",
+ " 'idx_': 2,\n",
+ " 'metadata': {},\n",
+ " 'outputs': [],\n",
+ " 'source': '#| echo: true \\nprint(3+4)'},\n",
+ " { 'cell_type': 'markdown',\n",
+ " 'idx_': 3,\n",
+ " 'metadata': {},\n",
+ " 'source': 'This file will become your README and also the index '\n",
+ " 'of your documentation.\\n'\n",
+ " '\\n'\n",
+ " '## Developer Guide\\n'\n",
+ " '\\n'\n",
+ " 'If you are new to using `nbdev` here are some useful '\n",
+ " 'pointers to get you started.\\n'\n",
+ " '\\n'\n",
+ " '### Install hamux_qmd in Development mode\\n'\n",
+ " '\\n'\n",
+ " '```sh\\n'\n",
+ " '# make sure hamux_qmd package is installed in '\n",
+ " 'development mode\\n'\n",
+ " '$ pip install -e .\\n'\n",
+ " '\\n'\n",
+ " '# make changes under nbs/ directory\\n'\n",
+ " '# ...\\n'\n",
+ " '\\n'\n",
+ " '# compile to have changes apply to hamux_qmd\\n'\n",
+ " '$ nbdev_prepare\\n'\n",
+ " '```\\n'\n",
+ " '\\n'\n",
+ " '## Usage\\n'\n",
+ " '\\n'\n",
+ " '### Installation\\n'\n",
+ " '\\n'\n",
+ " 'Install latest from the GitHub [repository][repo]:\\n'\n",
+ " '\\n'\n",
+ " '```sh\\n'\n",
+ " '$ pip install '\n",
+ " 'git+https://github.com/bhoov/hamux_qmd.git\\n'\n",
+ " '```\\n'\n",
+ " '\\n'\n",
+ " 'or from [conda][conda]\\n'\n",
+ " '\\n'\n",
+ " '```sh\\n'\n",
+ " '$ conda install -c bhoov hamux_qmd\\n'\n",
+ " '```\\n'\n",
+ " '\\n'\n",
+ " 'or from [pypi][pypi]\\n'\n",
+ " '\\n'\n",
+ " '\\n'\n",
+ " '```sh\\n'\n",
+ " '$ pip install hamux_qmd\\n'\n",
+ " '```\\n'\n",
+ " '\\n'\n",
+ " '\\n'\n",
+ " '[repo]: https://github.com/bhoov/hamux_qmd\\n'\n",
+ " '[docs]: https://bhoov.github.io/hamux_qmd/\\n'\n",
+ " '[pypi]: https://pypi.org/project/hamux_qmd/\\n'\n",
+ " '[conda]: https://anaconda.org/bhoov/hamux_qmd\\n'\n",
+ " '\\n'\n",
+ " '## How to use\\n'\n",
+ " '\\n'\n",
+ " \"Fill me in please! Don't forget code examples:\"},\n",
+ " { 'cell_type': 'code',\n",
+ " 'execution_count': None,\n",
+ " 'idx_': 4,\n",
+ " 'metadata': {},\n",
+ " 'outputs': [],\n",
+ " 'source': '1+1'}],\n",
+ " 'metadata': { 'kernelspec': { 'display_name': 'Python 3 (ipykernel)',\n",
+ " 'language': 'python',\n",
+ " 'name': 'python3',\n",
+ " 'path': '/Users/hoo/miniconda3/envs/nbdev/share/jupyter/kernels/python3'}},\n",
+ " 'nbformat': 4,\n",
+ " 'nbformat_minor': 4,\n",
+ " 'path_': '../../tests/tst_index.ipynb'}\n",
+ "```"
+ ],
+ "text/plain": [
+ "{'cells': [{'cell_type': 'markdown',\n",
+ " 'metadata': {},\n",
+ " 'source': '---\\ntitle: \"HAMUX QMD\"\\n---\\n\\n# HAMUX QMD\\n> Energy formulation for deep learning',\n",
+ " 'idx_': 0},\n",
+ " {'cell_type': 'code',\n",
+ " 'metadata': {},\n",
+ " 'source': '#| echo: false\\n# from hamux_qmd.core import *',\n",
+ " 'execution_count': None,\n",
+ " 'outputs': [],\n",
+ " 'idx_': 1},\n",
+ " {'cell_type': 'code',\n",
+ " 'metadata': {},\n",
+ " 'source': '#| echo: true \\nprint(3+4)',\n",
+ " 'execution_count': None,\n",
+ " 'outputs': [],\n",
+ " 'idx_': 2},\n",
+ " {'cell_type': 'markdown',\n",
+ " 'metadata': {},\n",
+ " 'source': \"This file will become your README and also the index of your documentation.\\n\\n## Developer Guide\\n\\nIf you are new to using `nbdev` here are some useful pointers to get you started.\\n\\n### Install hamux_qmd in Development mode\\n\\n```sh\\n# make sure hamux_qmd package is installed in development mode\\n$ pip install -e .\\n\\n# make changes under nbs/ directory\\n# ...\\n\\n# compile to have changes apply to hamux_qmd\\n$ nbdev_prepare\\n```\\n\\n## Usage\\n\\n### Installation\\n\\nInstall latest from the GitHub [repository][repo]:\\n\\n```sh\\n$ pip install git+https://github.com/bhoov/hamux_qmd.git\\n```\\n\\nor from [conda][conda]\\n\\n```sh\\n$ conda install -c bhoov hamux_qmd\\n```\\n\\nor from [pypi][pypi]\\n\\n\\n```sh\\n$ pip install hamux_qmd\\n```\\n\\n\\n[repo]: https://github.com/bhoov/hamux_qmd\\n[docs]: https://bhoov.github.io/hamux_qmd/\\n[pypi]: https://pypi.org/project/hamux_qmd/\\n[conda]: https://anaconda.org/bhoov/hamux_qmd\\n\\n## How to use\\n\\nFill me in please! Don't forget code examples:\",\n",
+ " 'idx_': 3},\n",
+ " {'cell_type': 'code',\n",
+ " 'metadata': {},\n",
+ " 'source': '1+1',\n",
+ " 'execution_count': None,\n",
+ " 'outputs': [],\n",
+ " 'idx_': 4}],\n",
+ " 'metadata': {'kernelspec': {'name': 'python3',\n",
+ " 'language': 'python',\n",
+ " 'display_name': 'Python 3 (ipykernel)',\n",
+ " 'path': '/Users/hoo/miniconda3/envs/nbdev/share/jupyter/kernels/python3'}},\n",
+ " 'nbformat': 4,\n",
+ " 'nbformat_minor': 4,\n",
+ " 'path_': '../../tests/tst_index.ipynb'}"
+ ]
+ },
+ "execution_count": null,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "read_nb(\"../../tests/tst_index.ipynb\")"
]
},
{
@@ -395,7 +849,7 @@
"outputs": [],
"source": [
"#|export\n",
- "def _read_nb_or_qmd(path):\n",
+ "def read_nb_or_qmd(path):\n",
" if Path(path).suffix == '.qmd': return read_qmd(path)\n",
" return read_nb(path)"
]
@@ -407,7 +861,7 @@
"metadata": {},
"outputs": [],
"source": [
- "out = _read_nb_or_qmd('../../tests/minimal.qmd')"
+ "out = read_nb_or_qmd('../../tests/minimal.qmd')"
]
},
{
@@ -421,7 +875,7 @@
"class NBProcessor:\n",
" \"Process cells and nbdev comments in a notebook\"\n",
" def __init__(self, path=None, procs=None, nb=None, debug=False, rm_directives=True, process=False):\n",
- " self.nb = _read_nb_or_qmd(path) if nb is None else nb\n",
+ " self.nb = read_nb_or_qmd(path) if nb is None else nb\n",
" self.lang = nb_lang(self.nb)\n",
" for cell in self.nb.cells: cell.directives_ = extract_directives(cell, remove=rm_directives, lang=self.lang)\n",
" self.procs = _mk_procs(procs, nb=self.nb)\n",
@@ -493,15 +947,45 @@
"def print_execs(cell):\n",
" if 'exec' in cell.source: print(cell.source)\n",
"\n",
- "NBProcessor(everything_fn, print_execs).process()"
+ "nbproc = NBProcessor(everything_fn, print_execs)\n",
+ "nbproc.process()"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "1645c63d",
+ "id": "4a6e10a9",
"metadata": {},
"outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1645c63d",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "---\n",
+ "title: Foo\n",
+ "execute:\n",
+ " echo: false\n",
+ "---\n",
+ "\n",
+ "# A Title\n",
+ "\n",
+ "> A description\n",
+ "\n",
+ "This notebook is used to demonstrate and test all the features of nbdev's export functionality. See the notebooks in `nbs` for how it's used.\n",
+ "exec(\"o_y=1\")\n",
+ "exec(\"p_y=1\")\n",
+ "_all_ = [o_y, 'p_y']\n"
+ ]
+ }
+ ],
"source": [
"everything_fn_qmd = '../../tests/01_everything.qmd'\n",
"\n",
diff --git a/nbs/api/14_quarto.ipynb b/nbs/api/14_quarto.ipynb
index 5ceabb923..a4cc90ecf 100644
--- a/nbs/api/14_quarto.ipynb
+++ b/nbs/api/14_quarto.ipynb
@@ -279,6 +279,7 @@
" css: styles.css\n",
" toc: true\n",
" keep-md: true\n",
+ " keep-ipynb: true\n",
" commonmark: default\n",
"\n",
"website:\n",
@@ -367,6 +368,7 @@
" nbdev.doclinks._build_modidx()\n",
" nbdev_sidebar.__wrapped__(path=path, **kwargs)\n",
" cache = proc_nbs(path, n_workers=n_workers, **kwargs)\n",
+ " print(\"cache: \", cache)\n",
" return cache,cfg,path"
]
},
@@ -486,7 +488,8 @@
"\n",
" with _SidebarYmlRemoved(path): # to avoid rendering whole website\n",
" cache = proc_nbs(path)\n",
- " _sprun(f'cd \"{cache}\" && quarto render \"{cache/cfg.readme_nb}\" -o README.md -t gfm --no-execute')\n",
+ " readme_nb_name = Path(cfg.readme_nb).with_suffix('.ipynb')\n",
+ " _sprun(f'cd \"{cache}\" && quarto render \"{cache/readme_nb_name}\" -o README.md -t gfm --no-execute')\n",
" \n",
" _save_cached_readme(cache, cfg)"
]
diff --git a/tests/tst_index.ipynb b/tests/tst_index.ipynb
new file mode 100644
index 000000000..0b1085f77
--- /dev/null
+++ b/tests/tst_index.ipynb
@@ -0,0 +1,112 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "---\n",
+ "title: \"HAMUX QMD\"\n",
+ "---\n",
+ "\n",
+ "# HAMUX QMD\n",
+ "> Energy formulation for deep learning"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "#| echo: false\n",
+ "# from hamux_qmd.core import *"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "#| echo: true \n",
+ "print(3+4)"
+ ],
+ "execution_count": null,
+ "outputs": []
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "This file will become your README and also the index of your documentation.\n",
+ "\n",
+ "## Developer Guide\n",
+ "\n",
+ "If you are new to using `nbdev` here are some useful pointers to get you started.\n",
+ "\n",
+ "### Install hamux_qmd in Development mode\n",
+ "\n",
+ "```sh\n",
+ "# make sure hamux_qmd package is installed in development mode\n",
+ "$ pip install -e .\n",
+ "\n",
+ "# make changes under nbs/ directory\n",
+ "# ...\n",
+ "\n",
+ "# compile to have changes apply to hamux_qmd\n",
+ "$ nbdev_prepare\n",
+ "```\n",
+ "\n",
+ "## Usage\n",
+ "\n",
+ "### Installation\n",
+ "\n",
+ "Install latest from the GitHub [repository][repo]:\n",
+ "\n",
+ "```sh\n",
+ "$ pip install git+https://github.com/bhoov/hamux_qmd.git\n",
+ "```\n",
+ "\n",
+ "or from [conda][conda]\n",
+ "\n",
+ "```sh\n",
+ "$ conda install -c bhoov hamux_qmd\n",
+ "```\n",
+ "\n",
+ "or from [pypi][pypi]\n",
+ "\n",
+ "\n",
+ "```sh\n",
+ "$ pip install hamux_qmd\n",
+ "```\n",
+ "\n",
+ "\n",
+ "[repo]: https://github.com/bhoov/hamux_qmd\n",
+ "[docs]: https://bhoov.github.io/hamux_qmd/\n",
+ "[pypi]: https://pypi.org/project/hamux_qmd/\n",
+ "[conda]: https://anaconda.org/bhoov/hamux_qmd\n",
+ "\n",
+ "## How to use\n",
+ "\n",
+ "Fill me in please! Don't forget code examples:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "metadata": {},
+ "source": [
+ "1+1"
+ ],
+ "execution_count": null,
+ "outputs": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "name": "python3",
+ "language": "python",
+ "display_name": "Python 3 (ipykernel)",
+ "path": "/Users/hoo/miniconda3/envs/nbdev/share/jupyter/kernels/python3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}
\ No newline at end of file
diff --git a/tests/tst_index.qmd b/tests/tst_index.qmd
new file mode 100644
index 000000000..afd29b157
--- /dev/null
+++ b/tests/tst_index.qmd
@@ -0,0 +1,74 @@
+---
+title: "HAMUX QMD"
+---
+
+# HAMUX QMD
+> Energy formulation for deep learning
+
+```{python}
+#| echo: false
+# from hamux_qmd.core import *
+```
+
+This is an insertion of another bit of markdown.
+
+``` {python .code-cell-2}
+#| echo: true
+print(6+7)
+```
+
+This file will become your README and also the index of your documentation.
+
+## Developer Guide
+
+If you are new to using `nbdev` here are some useful pointers to get you started.
+
+### Install hamux_qmd in Development mode
+
+```sh
+# make sure hamux_qmd package is installed in development mode
+$ pip install -e .
+
+# make changes under nbs/ directory
+# ...
+
+# compile to have changes apply to hamux_qmd
+$ nbdev_prepare
+```
+
+## Usage
+
+### Installation
+
+Install latest from the GitHub [repository][repo]:
+
+```sh
+$ pip install git+https://github.com/bhoov/hamux_qmd.git
+```
+
+or from [conda][conda]
+
+```sh
+$ conda install -c bhoov hamux_qmd
+```
+
+or from [pypi][pypi]
+
+
+```sh
+$ pip install hamux_qmd
+```
+
+
+[repo]: https://github.com/bhoov/hamux_qmd
+[docs]: https://bhoov.github.io/hamux_qmd/
+[pypi]: https://pypi.org/project/hamux_qmd/
+[conda]: https://anaconda.org/bhoov/hamux_qmd
+
+## How to use
+
+Fill me in please! Don't forget code examples:
+
+```{python}
+1+1
+```
\ No newline at end of file
diff --git a/tst_index_pandoc.ipynb b/tst_index_pandoc.ipynb
new file mode 100644
index 000000000..cbe085536
--- /dev/null
+++ b/tst_index_pandoc.ipynb
@@ -0,0 +1,82 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "------------------------------------------------------------------------\n",
+ "\n",
+ "## title: \"HAMUX QMD\" format: gfm+yaml_metadata_block\n",
+ "\n",
+ "# HAMUX\n",
+ "\n",
+ "``` {python}\n",
+ "#| echo: false\n",
+ "# from hamux_qmd.core import *\n",
+ "```\n",
+ "\n",
+ "> Energy formulation for deep learning\n",
+ "\n",
+ "This file will become your README and also the index of your\n",
+ "documentation.\n",
+ "\n",
+ "## Developer Guide\n",
+ "\n",
+ "If you are new to using `nbdev` here are some useful pointers to get you\n",
+ "started.\n",
+ "\n",
+ "### Install hamux_qmd in Development mode\n",
+ "\n",
+ "``` sh\n",
+ "# make sure hamux_qmd package is installed in development mode\n",
+ "$ pip install -e .\n",
+ "\n",
+ "# make changes under nbs/ directory\n",
+ "# ...\n",
+ "\n",
+ "# compile to have changes apply to hamux_qmd\n",
+ "$ nbdev_prepare\n",
+ "```\n",
+ "\n",
+ "## Usage\n",
+ "\n",
+ "### Installation\n",
+ "\n",
+ "Install latest from the GitHub\n",
+ "[repository](https://github.com/bhoov/hamux_qmd):\n",
+ "\n",
+ "``` sh\n",
+ "$ pip install git+https://github.com/bhoov/hamux_qmd.git\n",
+ "```\n",
+ "\n",
+ "or from [conda](https://anaconda.org/bhoov/hamux_qmd)\n",
+ "\n",
+ "``` sh\n",
+ "$ conda install -c bhoov hamux_qmd\n",
+ "```\n",
+ "\n",
+ "or from [pypi](https://pypi.org/project/hamux_qmd/)\n",
+ "\n",
+ "``` sh\n",
+ "$ pip install hamux_qmd\n",
+ "```\n",
+ "\n",
+ "## How to use\n",
+ "\n",
+ "Fill me in please! Don't forget code examples:\n",
+ "\n",
+ "``` {python}\n",
+ "1+1\n",
+ "```\n",
+ "\n",
+ "``` {python}\n",
+ "1+1 \n",
+ "```"
+ ],
+ "id": "f0de16ee-b9c9-4177-899b-0d78ff3d438e"
+ }
+ ],
+ "nbformat": 4,
+ "nbformat_minor": 5,
+ "metadata": {}
+}
From c47724913f316e89324ece39ab9e562262128d88 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Fri, 30 May 2025 12:34:56 -0400
Subject: [PATCH 07/31] Update sidebar.yml to look for .ipynb instead of .qmd
in cache folder
---
nbdev/quarto.py | 4 +++-
nbs/api/14_quarto.ipynb | 4 +++-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/nbdev/quarto.py b/nbdev/quarto.py
index 28a5448d7..968431c30 100644
--- a/nbdev/quarto.py
+++ b/nbdev/quarto.py
@@ -130,7 +130,9 @@ def _f(a,b): return Path(a),b
_dir = dir_struct
for subdir in drel.parts:
_dir = _dir.setdefault(subdir, dict())
- _dir[name] = name
+ if Path(name).suffix == '.qmd': name = Path(name).with_suffix('.ipynb') # .qmd files are converted to .ipynb before docs are rendered
+ print("new name: ", name)
+ _dir[name] = str(name)
_recursive_parser(dir_struct, _contents, Path())
yml_path = path/'sidebar.yml'
diff --git a/nbs/api/14_quarto.ipynb b/nbs/api/14_quarto.ipynb
index a4cc90ecf..b7b0ad8c6 100644
--- a/nbs/api/14_quarto.ipynb
+++ b/nbs/api/14_quarto.ipynb
@@ -234,7 +234,9 @@
" _dir = dir_struct\n",
" for subdir in drel.parts:\n",
" _dir = _dir.setdefault(subdir, dict())\n",
- " _dir[name] = name\n",
+ " if Path(name).suffix == '.qmd': name = Path(name).with_suffix('.ipynb') # .qmd files are converted to .ipynb before docs are rendered\n",
+ " print(\"new name: \", name)\n",
+ " _dir[name] = str(name)\n",
"\n",
" _recursive_parser(dir_struct, _contents, Path())\n",
" yml_path = path/'sidebar.yml'\n",
From 82d3205daa7a65bc63bad73868d7cc3c1331b900 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Fri, 30 May 2025 12:59:04 -0400
Subject: [PATCH 08/31] Execute .qmd files when rendering
---
nbdev/_modidx.py | 1 +
nbdev/serve_drv.py | 24 +++++++++++++++++-------
nbdev/test.py | 17 +++++++++--------
nbs/api/12_test.ipynb | 15 ++++++++-------
4 files changed, 35 insertions(+), 22 deletions(-)
diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py
index f1c2f3fb0..92c97137c 100644
--- a/nbdev/_modidx.py
+++ b/nbdev/_modidx.py
@@ -389,4 +389,5 @@
'nbdev.sync.nbdev_update': ('api/sync.html#nbdev_update', 'nbdev/sync.py')},
'nbdev.test': { 'nbdev.test._keep_file': ('api/test.html#_keep_file', 'nbdev/test.py'),
'nbdev.test.nbdev_test': ('api/test.html#nbdev_test', 'nbdev/test.py'),
+ 'nbdev.test.no_eval': ('api/test.html#no_eval', 'nbdev/test.py'),
'nbdev.test.test_nb': ('api/test.html#test_nb', 'nbdev/test.py')}}}
diff --git a/nbdev/serve_drv.py b/nbdev/serve_drv.py
index a20a5a1aa..f336d86d6 100644
--- a/nbdev/serve_drv.py
+++ b/nbdev/serve_drv.py
@@ -1,9 +1,13 @@
import os
from execnb.nbio import write_nb
-from nbdev.process import read_nb_or_qmd
+from execnb.shell import CaptureShell
+from nbdev.process import read_nb, read_qmd
from io import StringIO
from contextlib import redirect_stdout
-
+from fastcore.foundation import working_directory
+from nbdev.test import no_eval
+from nbdev.config import get_config
+
def exec_scr(src, dst, md):
f = StringIO()
g = {}
@@ -11,19 +15,25 @@ def exec_scr(src, dst, md):
res = ""
if md: res += "---\n" + md + "\n---\n\n"
dst.write_text(res + f.getvalue())
+
+def exec_qmd(src, dst, cb):
+ print(f"Executing QMD-derived notebook: {src}")
+ nb = read_qmd(src)
+ k = CaptureShell()
+ with working_directory(src.parent): k.run_all(nb, exc_stop=False, preproc=no_eval)
+ cb()(nb)
+ write_nb(nb, dst.with_suffix(".ipynb"))
def exec_nb(src, dst, cb):
- print("src", src)
- if src.suffix == ".qmd": dst = dst.with_suffix(".ipynb")
- print("dst", dst)
- nb = read_nb_or_qmd(src)
+ nb = read_nb(src)
cb()(nb)
write_nb(nb, dst)
def main(o):
src,dst,x = o
os.environ["IN_TEST"] = "1"
- if src.suffix in ['.ipynb','.qmd']: exec_nb(src, dst, x)
+ if src.suffix=='.ipynb': exec_nb(src, dst, x)
+ elif src.suffix=='.qmd': exec_qmd(src, dst, x)
elif src.suffix=='.py': exec_scr(src, dst, x)
else: raise Exception(src)
del os.environ["IN_TEST"]
diff --git a/nbdev/test.py b/nbdev/test.py
index ee223232a..0ab56c1a0 100644
--- a/nbdev/test.py
+++ b/nbdev/test.py
@@ -3,7 +3,7 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/12_test.ipynb.
# %% auto 0
-__all__ = ['test_nb', 'nbdev_test']
+__all__ = ['no_eval', 'test_nb', 'nbdev_test']
# %% ../nbs/api/12_test.ipynb
import time,os,sys,traceback,contextlib, inspect
@@ -23,6 +23,13 @@
from execnb.shell import *
# %% ../nbs/api/12_test.ipynb
+def no_eval(cell, flags=set()):
+ if cell.cell_type != 'code': return True
+ if 'nbdev_export'+'(' in cell.source: return True
+ direc = getattr(cell, 'directives_', {}) or {}
+ if direc.get('eval:', [''])[0].lower() == 'false': return True
+ return flags & direc.keys()
+
def test_nb(fn, # file name of notebook to test
skip_flags=None, # list of flags marking cells to skip
force_flags=None, # list of flags marking cells to always run
@@ -33,17 +40,11 @@ def test_nb(fn, # file name of notebook to test
if basepath: sys.path.insert(0, str(basepath))
if not IN_NOTEBOOK: os.environ["IN_TEST"] = '1'
flags=set(L(skip_flags)) - set(L(force_flags))
+ _no_eval = partial(no_eval, flags=flags)
nb = NBProcessor(fn, procs=FrontmatterProc, process=True).nb
fm = getattr(nb, 'frontmatter_', {})
if str2bool(fm.get('skip_exec', False)) or nb_lang(nb) != 'python': return True, 0
- def _no_eval(cell):
- if cell.cell_type != 'code': return True
- if 'nbdev_export'+'(' in cell.source: return True
- direc = getattr(cell, 'directives_', {}) or {}
- if direc.get('eval:', [''])[0].lower() == 'false': return True
- return flags & direc.keys()
-
start = time.time()
k = CaptureShell(fn)
if do_print: print(f'Starting {fn}')
diff --git a/nbs/api/12_test.ipynb b/nbs/api/12_test.ipynb
index 5cbb90f12..57000d101 100644
--- a/nbs/api/12_test.ipynb
+++ b/nbs/api/12_test.ipynb
@@ -53,6 +53,13 @@
"outputs": [],
"source": [
"#|export\n",
+ "def no_eval(cell, flags=set()):\n",
+ " if cell.cell_type != 'code': return True\n",
+ " if 'nbdev_export'+'(' in cell.source: return True\n",
+ " direc = getattr(cell, 'directives_', {}) or {}\n",
+ " if direc.get('eval:', [''])[0].lower() == 'false': return True\n",
+ " return flags & direc.keys()\n",
+ "\n",
"def test_nb(fn, # file name of notebook to test\n",
" skip_flags=None, # list of flags marking cells to skip\n",
" force_flags=None, # list of flags marking cells to always run\n",
@@ -63,17 +70,11 @@
" if basepath: sys.path.insert(0, str(basepath))\n",
" if not IN_NOTEBOOK: os.environ[\"IN_TEST\"] = '1'\n",
" flags=set(L(skip_flags)) - set(L(force_flags))\n",
+ " _no_eval = partial(no_eval, flags=flags)\n",
" nb = NBProcessor(fn, procs=FrontmatterProc, process=True).nb\n",
" fm = getattr(nb, 'frontmatter_', {})\n",
" if str2bool(fm.get('skip_exec', False)) or nb_lang(nb) != 'python': return True, 0\n",
"\n",
- " def _no_eval(cell):\n",
- " if cell.cell_type != 'code': return True\n",
- " if 'nbdev_export'+'(' in cell.source: return True\n",
- " direc = getattr(cell, 'directives_', {}) or {}\n",
- " if direc.get('eval:', [''])[0].lower() == 'false': return True\n",
- " return flags & direc.keys()\n",
- " \n",
" start = time.time()\n",
" k = CaptureShell(fn)\n",
" if do_print: print(f'Starting {fn}')\n",
From b98f5ee5e852a3c8680616f5f08ec4802c1b97c1 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Fri, 30 May 2025 15:37:54 -0400
Subject: [PATCH 09/31] Cleanup function in succinct style of nbdev
---
nbdev/process.py | 34 +---
nbs/api/03_process.ipynb | 412 +++------------------------------------
tests/tst_index.ipynb | 59 +++---
tests/tst_index.qmd | 37 ++--
4 files changed, 77 insertions(+), 465 deletions(-)
diff --git a/nbdev/process.py b/nbdev/process.py
index 44bb19716..b5bfb8b7a 100644
--- a/nbdev/process.py
+++ b/nbdev/process.py
@@ -19,51 +19,33 @@
# %% ../nbs/api/03_process.ipynb
def _qmd_to_raw_cell(source_str, cell_type_str):
- """Helper to create a raw cell dictionary (like from a .ipynb JSON)."""
- # Source is typically a list of strings in .ipynb, but a single string is also common.
- # NbCell's set_source method handles both by doing ''.join(source).
- # Here, we'll provide it as a single string, which is what re.split gives us.
- cell = {
- 'cell_type': cell_type_str,
- 'metadata': {}, # Standard empty metadata
- 'source': source_str # Already stripped by the caller
- }
+ """Create a default ipynb json cell"""
+ cell = {'cell_type': cell_type_str, 'metadata': {}, 'source': source_str}
if cell_type_str == 'code':
- cell['execution_count'] = None # In JSON, this is null
+ cell['execution_count'] = None
cell['outputs'] = []
return cell
def read_qmd(path): # Renamed to clarify output
- """
- Reads a .qmd file and returns a notebook structure as a JSON-like dictionary,
- compatible with execnb.nbio.dict2nb.
- """
+ """Reads a .qmd file as an nb compatible with the rest of execnb and nbdev"""
content = Path(path).read_text(encoding='utf-8')
cell_pat = re.compile(r"^(`{3,})\s*\{python[^\n]*\}\s*(.*?)^\1\s*$", re.MULTILINE | re.DOTALL)
- # parts will be [md_chunk, captured_backticks_1, captured_code_1, md_chunk_2, ...]
+ # `parts` will be [md_chunk, captured_backticks_1, captured_code_1, md_chunk_2, ...]
+ # We just care about md and code chunks
parts = cell_pat.split(content)
raw_cells = []
# Handle the first markdown segment (before any code cells, or all content if no code cells)
initial_md_source = parts[0].strip()
if initial_md_source: raw_cells.append(_qmd_to_raw_cell(initial_md_source, 'markdown'))
-
- # Process subsequent parts: captured code (parts[i+1]) and following markdown (parts[i+2])
- # Loop from the first captured group (backticks, at index 1) with a step of 3.
for i in range(1, len(parts), 3):
- # Code cell content is in parts[i+1] (second capture group)
if i + 1 < len(parts):
code_source = parts[i+1].strip()
- # Only add non-empty code cells, though regex usually ensures content.
- if code_source:
- raw_cells.append(_qmd_to_raw_cell(code_source, 'code'))
-
- # Markdown cell content following the code cell is in parts[i+2]
+ if code_source: raw_cells.append(_qmd_to_raw_cell(code_source, 'code'))
if i + 2 < len(parts):
intermediate_md_source = parts[i+2].strip()
- if intermediate_md_source:
- raw_cells.append(_qmd_to_raw_cell(intermediate_md_source, 'markdown'))
+ if intermediate_md_source: raw_cells.append(_qmd_to_raw_cell(intermediate_md_source, 'markdown'))
# Construct the final notebook dictionary
notebook_dict = {
diff --git a/nbs/api/03_process.ipynb b/nbs/api/03_process.ipynb
index fc4f703b8..1ec668d32 100644
--- a/nbs/api/03_process.ipynb
+++ b/nbs/api/03_process.ipynb
@@ -81,51 +81,33 @@
"source": [
"#|export\n",
"def _qmd_to_raw_cell(source_str, cell_type_str):\n",
- " \"\"\"Helper to create a raw cell dictionary (like from a .ipynb JSON).\"\"\"\n",
- " # Source is typically a list of strings in .ipynb, but a single string is also common.\n",
- " # NbCell's set_source method handles both by doing ''.join(source).\n",
- " # Here, we'll provide it as a single string, which is what re.split gives us.\n",
- " cell = {\n",
- " 'cell_type': cell_type_str,\n",
- " 'metadata': {}, # Standard empty metadata\n",
- " 'source': source_str # Already stripped by the caller\n",
- " }\n",
+ " \"\"\"Create a default ipynb json cell\"\"\"\n",
+ " cell = {'cell_type': cell_type_str, 'metadata': {}, 'source': source_str}\n",
" if cell_type_str == 'code':\n",
- " cell['execution_count'] = None # In JSON, this is null\n",
+ " cell['execution_count'] = None\n",
" cell['outputs'] = []\n",
" return cell\n",
"\n",
"def read_qmd(path): # Renamed to clarify output\n",
- " \"\"\"\n",
- " Reads a .qmd file and returns a notebook structure as a JSON-like dictionary,\n",
- " compatible with execnb.nbio.dict2nb.\n",
- " \"\"\"\n",
+ " \"\"\"Reads a .qmd file as an nb compatible with the rest of execnb and nbdev\"\"\"\n",
" content = Path(path).read_text(encoding='utf-8')\n",
" cell_pat = re.compile(r\"^(`{3,})\\s*\\{python[^\\n]*\\}\\s*(.*?)^\\1\\s*$\", re.MULTILINE | re.DOTALL)\n",
" \n",
- " # parts will be [md_chunk, captured_backticks_1, captured_code_1, md_chunk_2, ...]\n",
+ " # `parts` will be [md_chunk, captured_backticks_1, captured_code_1, md_chunk_2, ...]\n",
+ " # We just care about md and code chunks\n",
" parts = cell_pat.split(content)\n",
" raw_cells = []\n",
" \n",
" # Handle the first markdown segment (before any code cells, or all content if no code cells)\n",
" initial_md_source = parts[0].strip()\n",
" if initial_md_source: raw_cells.append(_qmd_to_raw_cell(initial_md_source, 'markdown'))\n",
- " \n",
- " # Process subsequent parts: captured code (parts[i+1]) and following markdown (parts[i+2])\n",
- " # Loop from the first captured group (backticks, at index 1) with a step of 3.\n",
" for i in range(1, len(parts), 3):\n",
- " # Code cell content is in parts[i+1] (second capture group)\n",
" if i + 1 < len(parts):\n",
" code_source = parts[i+1].strip()\n",
- " # Only add non-empty code cells, though regex usually ensures content.\n",
- " if code_source: \n",
- " raw_cells.append(_qmd_to_raw_cell(code_source, 'code'))\n",
- " \n",
- " # Markdown cell content following the code cell is in parts[i+2]\n",
+ " if code_source: raw_cells.append(_qmd_to_raw_cell(code_source, 'code'))\n",
" if i + 2 < len(parts):\n",
" intermediate_md_source = parts[i+2].strip()\n",
- " if intermediate_md_source:\n",
- " raw_cells.append(_qmd_to_raw_cell(intermediate_md_source, 'markdown'))\n",
+ " if intermediate_md_source: raw_cells.append(_qmd_to_raw_cell(intermediate_md_source, 'markdown'))\n",
" \n",
" # Construct the final notebook dictionary\n",
" notebook_dict = {\n",
@@ -148,64 +130,12 @@
"execution_count": null,
"id": "1559503a",
"metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "ipynb: 5\n"
- ]
- },
- {
- "data": {
- "text/plain": [
- "[{'cell_type': 'markdown',\n",
- " 'metadata': {},\n",
- " 'source': '---\\ntitle: \"HAMUX QMD\"\\n---\\n\\n# HAMUX QMD\\n> Energy formulation for deep learning',\n",
- " 'idx_': 0,\n",
- " 'directives_': {}},\n",
- " {'cell_type': 'code',\n",
- " 'metadata': {},\n",
- " 'source': '#| echo: false\\n# from hamux_qmd.core import *',\n",
- " 'execution_count': None,\n",
- " 'outputs': [],\n",
- " 'idx_': 1,\n",
- " 'directives_': {'echo:': ['false']}},\n",
- " {'cell_type': 'code',\n",
- " 'metadata': {},\n",
- " 'source': '#| echo: true \\nprint(3+4)',\n",
- " 'execution_count': None,\n",
- " 'outputs': [],\n",
- " 'idx_': 2,\n",
- " 'directives_': {'echo:': ['true']}},\n",
- " {'cell_type': 'markdown',\n",
- " 'metadata': {},\n",
- " 'source': \"This file will become your README and also the index of your documentation.\\n\\n## Developer Guide\\n\\nIf you are new to using `nbdev` here are some useful pointers to get you started.\\n\\n### Install hamux_qmd in Development mode\\n\\n```sh\\n# make sure hamux_qmd package is installed in development mode\\n$ pip install -e .\\n\\n# make changes under nbs/ directory\\n# ...\\n\\n# compile to have changes apply to hamux_qmd\\n$ nbdev_prepare\\n```\\n\\n## Usage\\n\\n### Installation\\n\\nInstall latest from the GitHub [repository][repo]:\\n\\n```sh\\n$ pip install git+https://github.com/bhoov/hamux_qmd.git\\n```\\n\\nor from [conda][conda]\\n\\n```sh\\n$ conda install -c bhoov hamux_qmd\\n```\\n\\nor from [pypi][pypi]\\n\\n\\n```sh\\n$ pip install hamux_qmd\\n```\\n\\n\\n[repo]: https://github.com/bhoov/hamux_qmd\\n[docs]: https://bhoov.github.io/hamux_qmd/\\n[pypi]: https://pypi.org/project/hamux_qmd/\\n[conda]: https://anaconda.org/bhoov/hamux_qmd\\n\\n## How to use\\n\\nFill me in please! Don't forget code examples:\",\n",
- " 'idx_': 3,\n",
- " 'directives_': {}},\n",
- " {'cell_type': 'code',\n",
- " 'metadata': {},\n",
- " 'source': '1+1',\n",
- " 'execution_count': None,\n",
- " 'outputs': [],\n",
- " 'idx_': 4,\n",
- " 'directives_': {}}]"
- ]
- },
- "execution_count": null,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
+ "outputs": [],
"source": [
- "# Test: I want the .qmd file to be the exact same .ipynb file in the nb source (except for outputs)\n",
- "from nbdev.process import NBProcessor\n",
- "tst_ipynb = '../../tests/tst_index.ipynb'\n",
- "tst_qmd = '../../tests/tst_index.qmd'\n",
- "\n",
- "nbproc_ipynb = NBProcessor(tst_ipynb)\n",
- "nbproc_qmd = NBProcessor(tst_qmd)\n",
- "nbproc_ipynb.nb.cells"
+ "#| hide\n",
+ "# Test: we want the .qmd file to be very close to .ipynb file in the nb source (except for saved outputs which are not present in .qmd)\n",
+ "nb_qmd = read_qmd(\"../../tests/tst_index.qmd\")\n",
+ "nb_ipynb = read_nb(\"../../tests/tst_index.ipynb\")"
]
},
{
@@ -225,6 +155,7 @@
}
],
"source": [
+ "#| hide\n",
"tst_cell = \"\"\"\\\n",
"# Title \n",
"> description\n",
@@ -251,315 +182,18 @@
"\"\"\"\n",
"cell_pat = re.compile(r\"^(`{3,})\\s*\\{python[^\\n]*\\}\\s*(.*?)^\\1\\s*$\", re.MULTILINE | re.DOTALL)\n",
"matches = cell_pat.findall(tst_cell)\n",
+ "assert len(matches) == 3\n",
+ "assert len(matches[0][0]) == 3 # 3 backticks\n",
+ "assert len(matches[1][0]) == 4 # 4 backticks\n",
+ "assert len(matches[2][0]) == 5 # 5 backticks\n",
+ "\n",
+ "assert matches[0][1] == '#| export\\nprint(3+4)\\n'\n",
+ "assert matches[1][1] == '#| export\\nprint(9+12)\\n'\n",
+ "assert matches[2][1] == 'print(5+6)\\n'\n",
+ "\n",
"for match in matches: print(match)\n"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "a315bb32",
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/markdown": [
- "```json\n",
- "{ 'cells': [ { 'cell_type': 'markdown',\n",
- " 'idx_': 0,\n",
- " 'metadata': {},\n",
- " 'source': '---\\n'\n",
- " 'title: \"HAMUX QMD\"\\n'\n",
- " '---\\n'\n",
- " '\\n'\n",
- " '# HAMUX QMD\\n'\n",
- " '> Energy formulation for deep learning'},\n",
- " { 'cell_type': 'code',\n",
- " 'execution_count': None,\n",
- " 'idx_': 1,\n",
- " 'metadata': {},\n",
- " 'outputs': [],\n",
- " 'source': '#| echo: false\\n# from hamux_qmd.core import *'},\n",
- " { 'cell_type': 'markdown',\n",
- " 'idx_': 2,\n",
- " 'metadata': {},\n",
- " 'source': 'This is an insertion of another bit of markdown.'},\n",
- " { 'cell_type': 'code',\n",
- " 'execution_count': None,\n",
- " 'idx_': 3,\n",
- " 'metadata': {},\n",
- " 'outputs': [],\n",
- " 'source': '#| echo: true\\nprint(6+7)'},\n",
- " { 'cell_type': 'markdown',\n",
- " 'idx_': 4,\n",
- " 'metadata': {},\n",
- " 'source': 'This file will become your README and also the index '\n",
- " 'of your documentation.\\n'\n",
- " '\\n'\n",
- " '## Developer Guide\\n'\n",
- " '\\n'\n",
- " 'If you are new to using `nbdev` here are some useful '\n",
- " 'pointers to get you started.\\n'\n",
- " '\\n'\n",
- " '### Install hamux_qmd in Development mode\\n'\n",
- " '\\n'\n",
- " '```sh\\n'\n",
- " '# make sure hamux_qmd package is installed in '\n",
- " 'development mode\\n'\n",
- " '$ pip install -e .\\n'\n",
- " '\\n'\n",
- " '# make changes under nbs/ directory\\n'\n",
- " '# ...\\n'\n",
- " '\\n'\n",
- " '# compile to have changes apply to hamux_qmd\\n'\n",
- " '$ nbdev_prepare\\n'\n",
- " '```\\n'\n",
- " '\\n'\n",
- " '## Usage\\n'\n",
- " '\\n'\n",
- " '### Installation\\n'\n",
- " '\\n'\n",
- " 'Install latest from the GitHub [repository][repo]:\\n'\n",
- " '\\n'\n",
- " '```sh\\n'\n",
- " '$ pip install '\n",
- " 'git+https://github.com/bhoov/hamux_qmd.git\\n'\n",
- " '```\\n'\n",
- " '\\n'\n",
- " 'or from [conda][conda]\\n'\n",
- " '\\n'\n",
- " '```sh\\n'\n",
- " '$ conda install -c bhoov hamux_qmd\\n'\n",
- " '```\\n'\n",
- " '\\n'\n",
- " 'or from [pypi][pypi]\\n'\n",
- " '\\n'\n",
- " '\\n'\n",
- " '```sh\\n'\n",
- " '$ pip install hamux_qmd\\n'\n",
- " '```\\n'\n",
- " '\\n'\n",
- " '\\n'\n",
- " '[repo]: https://github.com/bhoov/hamux_qmd\\n'\n",
- " '[docs]: https://bhoov.github.io/hamux_qmd/\\n'\n",
- " '[pypi]: https://pypi.org/project/hamux_qmd/\\n'\n",
- " '[conda]: https://anaconda.org/bhoov/hamux_qmd\\n'\n",
- " '\\n'\n",
- " '## How to use\\n'\n",
- " '\\n'\n",
- " \"Fill me in please! Don't forget code examples:\"},\n",
- " { 'cell_type': 'code',\n",
- " 'execution_count': None,\n",
- " 'idx_': 5,\n",
- " 'metadata': {},\n",
- " 'outputs': [],\n",
- " 'source': '1+1'}],\n",
- " 'metadata': { 'kernelspec': { 'display_name': 'Python 3',\n",
- " 'language': 'python',\n",
- " 'name': 'python3'},\n",
- " 'language_info': {'name': 'python'},\n",
- " 'path_': '../../tests/tst_index.qmd'},\n",
- " 'nbformat': 4,\n",
- " 'nbformat_minor': 5}\n",
- "```"
- ],
- "text/plain": [
- "{'cells': [{'cell_type': 'markdown',\n",
- " 'metadata': {},\n",
- " 'source': '---\\ntitle: \"HAMUX QMD\"\\n---\\n\\n# HAMUX QMD\\n> Energy formulation for deep learning',\n",
- " 'idx_': 0},\n",
- " {'cell_type': 'code',\n",
- " 'metadata': {},\n",
- " 'source': '#| echo: false\\n# from hamux_qmd.core import *',\n",
- " 'execution_count': None,\n",
- " 'outputs': [],\n",
- " 'idx_': 1},\n",
- " {'cell_type': 'markdown',\n",
- " 'metadata': {},\n",
- " 'source': 'This is an insertion of another bit of markdown.',\n",
- " 'idx_': 2},\n",
- " {'cell_type': 'code',\n",
- " 'metadata': {},\n",
- " 'source': '#| echo: true\\nprint(6+7)',\n",
- " 'execution_count': None,\n",
- " 'outputs': [],\n",
- " 'idx_': 3},\n",
- " {'cell_type': 'markdown',\n",
- " 'metadata': {},\n",
- " 'source': \"This file will become your README and also the index of your documentation.\\n\\n## Developer Guide\\n\\nIf you are new to using `nbdev` here are some useful pointers to get you started.\\n\\n### Install hamux_qmd in Development mode\\n\\n```sh\\n# make sure hamux_qmd package is installed in development mode\\n$ pip install -e .\\n\\n# make changes under nbs/ directory\\n# ...\\n\\n# compile to have changes apply to hamux_qmd\\n$ nbdev_prepare\\n```\\n\\n## Usage\\n\\n### Installation\\n\\nInstall latest from the GitHub [repository][repo]:\\n\\n```sh\\n$ pip install git+https://github.com/bhoov/hamux_qmd.git\\n```\\n\\nor from [conda][conda]\\n\\n```sh\\n$ conda install -c bhoov hamux_qmd\\n```\\n\\nor from [pypi][pypi]\\n\\n\\n```sh\\n$ pip install hamux_qmd\\n```\\n\\n\\n[repo]: https://github.com/bhoov/hamux_qmd\\n[docs]: https://bhoov.github.io/hamux_qmd/\\n[pypi]: https://pypi.org/project/hamux_qmd/\\n[conda]: https://anaconda.org/bhoov/hamux_qmd\\n\\n## How to use\\n\\nFill me in please! Don't forget code examples:\",\n",
- " 'idx_': 4},\n",
- " {'cell_type': 'code',\n",
- " 'metadata': {},\n",
- " 'source': '1+1',\n",
- " 'execution_count': None,\n",
- " 'outputs': [],\n",
- " 'idx_': 5}],\n",
- " 'metadata': {'kernelspec': {'display_name': 'Python 3',\n",
- " 'language': 'python',\n",
- " 'name': 'python3'},\n",
- " 'language_info': {'name': 'python'},\n",
- " 'path_': '../../tests/tst_index.qmd'},\n",
- " 'nbformat': 4,\n",
- " 'nbformat_minor': 5}"
- ]
- },
- "execution_count": null,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "read_qmd(\"../../tests/tst_index.qmd\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "a9481494",
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/markdown": [
- "```json\n",
- "{ 'cells': [ { 'cell_type': 'markdown',\n",
- " 'idx_': 0,\n",
- " 'metadata': {},\n",
- " 'source': '---\\n'\n",
- " 'title: \"HAMUX QMD\"\\n'\n",
- " '---\\n'\n",
- " '\\n'\n",
- " '# HAMUX QMD\\n'\n",
- " '> Energy formulation for deep learning'},\n",
- " { 'cell_type': 'code',\n",
- " 'execution_count': None,\n",
- " 'idx_': 1,\n",
- " 'metadata': {},\n",
- " 'outputs': [],\n",
- " 'source': '#| echo: false\\n# from hamux_qmd.core import *'},\n",
- " { 'cell_type': 'code',\n",
- " 'execution_count': None,\n",
- " 'idx_': 2,\n",
- " 'metadata': {},\n",
- " 'outputs': [],\n",
- " 'source': '#| echo: true \\nprint(3+4)'},\n",
- " { 'cell_type': 'markdown',\n",
- " 'idx_': 3,\n",
- " 'metadata': {},\n",
- " 'source': 'This file will become your README and also the index '\n",
- " 'of your documentation.\\n'\n",
- " '\\n'\n",
- " '## Developer Guide\\n'\n",
- " '\\n'\n",
- " 'If you are new to using `nbdev` here are some useful '\n",
- " 'pointers to get you started.\\n'\n",
- " '\\n'\n",
- " '### Install hamux_qmd in Development mode\\n'\n",
- " '\\n'\n",
- " '```sh\\n'\n",
- " '# make sure hamux_qmd package is installed in '\n",
- " 'development mode\\n'\n",
- " '$ pip install -e .\\n'\n",
- " '\\n'\n",
- " '# make changes under nbs/ directory\\n'\n",
- " '# ...\\n'\n",
- " '\\n'\n",
- " '# compile to have changes apply to hamux_qmd\\n'\n",
- " '$ nbdev_prepare\\n'\n",
- " '```\\n'\n",
- " '\\n'\n",
- " '## Usage\\n'\n",
- " '\\n'\n",
- " '### Installation\\n'\n",
- " '\\n'\n",
- " 'Install latest from the GitHub [repository][repo]:\\n'\n",
- " '\\n'\n",
- " '```sh\\n'\n",
- " '$ pip install '\n",
- " 'git+https://github.com/bhoov/hamux_qmd.git\\n'\n",
- " '```\\n'\n",
- " '\\n'\n",
- " 'or from [conda][conda]\\n'\n",
- " '\\n'\n",
- " '```sh\\n'\n",
- " '$ conda install -c bhoov hamux_qmd\\n'\n",
- " '```\\n'\n",
- " '\\n'\n",
- " 'or from [pypi][pypi]\\n'\n",
- " '\\n'\n",
- " '\\n'\n",
- " '```sh\\n'\n",
- " '$ pip install hamux_qmd\\n'\n",
- " '```\\n'\n",
- " '\\n'\n",
- " '\\n'\n",
- " '[repo]: https://github.com/bhoov/hamux_qmd\\n'\n",
- " '[docs]: https://bhoov.github.io/hamux_qmd/\\n'\n",
- " '[pypi]: https://pypi.org/project/hamux_qmd/\\n'\n",
- " '[conda]: https://anaconda.org/bhoov/hamux_qmd\\n'\n",
- " '\\n'\n",
- " '## How to use\\n'\n",
- " '\\n'\n",
- " \"Fill me in please! Don't forget code examples:\"},\n",
- " { 'cell_type': 'code',\n",
- " 'execution_count': None,\n",
- " 'idx_': 4,\n",
- " 'metadata': {},\n",
- " 'outputs': [],\n",
- " 'source': '1+1'}],\n",
- " 'metadata': { 'kernelspec': { 'display_name': 'Python 3 (ipykernel)',\n",
- " 'language': 'python',\n",
- " 'name': 'python3',\n",
- " 'path': '/Users/hoo/miniconda3/envs/nbdev/share/jupyter/kernels/python3'}},\n",
- " 'nbformat': 4,\n",
- " 'nbformat_minor': 4,\n",
- " 'path_': '../../tests/tst_index.ipynb'}\n",
- "```"
- ],
- "text/plain": [
- "{'cells': [{'cell_type': 'markdown',\n",
- " 'metadata': {},\n",
- " 'source': '---\\ntitle: \"HAMUX QMD\"\\n---\\n\\n# HAMUX QMD\\n> Energy formulation for deep learning',\n",
- " 'idx_': 0},\n",
- " {'cell_type': 'code',\n",
- " 'metadata': {},\n",
- " 'source': '#| echo: false\\n# from hamux_qmd.core import *',\n",
- " 'execution_count': None,\n",
- " 'outputs': [],\n",
- " 'idx_': 1},\n",
- " {'cell_type': 'code',\n",
- " 'metadata': {},\n",
- " 'source': '#| echo: true \\nprint(3+4)',\n",
- " 'execution_count': None,\n",
- " 'outputs': [],\n",
- " 'idx_': 2},\n",
- " {'cell_type': 'markdown',\n",
- " 'metadata': {},\n",
- " 'source': \"This file will become your README and also the index of your documentation.\\n\\n## Developer Guide\\n\\nIf you are new to using `nbdev` here are some useful pointers to get you started.\\n\\n### Install hamux_qmd in Development mode\\n\\n```sh\\n# make sure hamux_qmd package is installed in development mode\\n$ pip install -e .\\n\\n# make changes under nbs/ directory\\n# ...\\n\\n# compile to have changes apply to hamux_qmd\\n$ nbdev_prepare\\n```\\n\\n## Usage\\n\\n### Installation\\n\\nInstall latest from the GitHub [repository][repo]:\\n\\n```sh\\n$ pip install git+https://github.com/bhoov/hamux_qmd.git\\n```\\n\\nor from [conda][conda]\\n\\n```sh\\n$ conda install -c bhoov hamux_qmd\\n```\\n\\nor from [pypi][pypi]\\n\\n\\n```sh\\n$ pip install hamux_qmd\\n```\\n\\n\\n[repo]: https://github.com/bhoov/hamux_qmd\\n[docs]: https://bhoov.github.io/hamux_qmd/\\n[pypi]: https://pypi.org/project/hamux_qmd/\\n[conda]: https://anaconda.org/bhoov/hamux_qmd\\n\\n## How to use\\n\\nFill me in please! Don't forget code examples:\",\n",
- " 'idx_': 3},\n",
- " {'cell_type': 'code',\n",
- " 'metadata': {},\n",
- " 'source': '1+1',\n",
- " 'execution_count': None,\n",
- " 'outputs': [],\n",
- " 'idx_': 4}],\n",
- " 'metadata': {'kernelspec': {'name': 'python3',\n",
- " 'language': 'python',\n",
- " 'display_name': 'Python 3 (ipykernel)',\n",
- " 'path': '/Users/hoo/miniconda3/envs/nbdev/share/jupyter/kernels/python3'}},\n",
- " 'nbformat': 4,\n",
- " 'nbformat_minor': 4,\n",
- " 'path_': '../../tests/tst_index.ipynb'}"
- ]
- },
- "execution_count": null,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "read_nb(\"../../tests/tst_index.ipynb\")"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
diff --git a/tests/tst_index.ipynb b/tests/tst_index.ipynb
index 0b1085f77..dd018550c 100644
--- a/tests/tst_index.ipynb
+++ b/tests/tst_index.ipynb
@@ -5,32 +5,37 @@
"metadata": {},
"source": [
"---\n",
- "title: \"HAMUX QMD\"\n",
- "---\n",
- "\n",
- "# HAMUX QMD\n",
- "> Energy formulation for deep learning"
+ "title: Example Project (from YAML)\n",
+ "---"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Example Project\n",
+ "> Example project"
]
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
"#| echo: false\n",
- "# from hamux_qmd.core import *"
- ],
- "execution_count": null,
- "outputs": []
+ "# from example_proj.core import *"
+ ]
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
"#| echo: true \n",
"print(3+4)"
- ],
- "execution_count": null,
- "outputs": []
+ ]
},
{
"cell_type": "markdown",
@@ -42,16 +47,16 @@
"\n",
"If you are new to using `nbdev` here are some useful pointers to get you started.\n",
"\n",
- "### Install hamux_qmd in Development mode\n",
+ "### Install example_proj in Development mode\n",
"\n",
"```sh\n",
- "# make sure hamux_qmd package is installed in development mode\n",
+ "# make sure example_proj package is installed in development mode\n",
"$ pip install -e .\n",
"\n",
"# make changes under nbs/ directory\n",
"# ...\n",
"\n",
- "# compile to have changes apply to hamux_qmd\n",
+ "# compile to have changes apply to example_proj\n",
"$ nbdev_prepare\n",
"```\n",
"\n",
@@ -62,28 +67,22 @@
"Install latest from the GitHub [repository][repo]:\n",
"\n",
"```sh\n",
- "$ pip install git+https://github.com/bhoov/hamux_qmd.git\n",
+ "$ pip install ...\n",
"```\n",
"\n",
"or from [conda][conda]\n",
"\n",
"```sh\n",
- "$ conda install -c bhoov hamux_qmd\n",
+ "$ conda install -c ...\n",
"```\n",
"\n",
"or from [pypi][pypi]\n",
"\n",
"\n",
"```sh\n",
- "$ pip install hamux_qmd\n",
+ "$ pip install ...\n",
"```\n",
"\n",
- "\n",
- "[repo]: https://github.com/bhoov/hamux_qmd\n",
- "[docs]: https://bhoov.github.io/hamux_qmd/\n",
- "[pypi]: https://pypi.org/project/hamux_qmd/\n",
- "[conda]: https://anaconda.org/bhoov/hamux_qmd\n",
- "\n",
"## How to use\n",
"\n",
"Fill me in please! Don't forget code examples:"
@@ -91,22 +90,22 @@
},
{
"cell_type": "code",
+ "execution_count": null,
"metadata": {},
+ "outputs": [],
"source": [
"1+1"
- ],
- "execution_count": null,
- "outputs": []
+ ]
}
],
"metadata": {
"kernelspec": {
- "name": "python3",
- "language": "python",
"display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3",
"path": "/Users/hoo/miniconda3/envs/nbdev/share/jupyter/kernels/python3"
}
},
"nbformat": 4,
"nbformat_minor": 4
-}
\ No newline at end of file
+}
diff --git a/tests/tst_index.qmd b/tests/tst_index.qmd
index afd29b157..c3ee1cad2 100644
--- a/tests/tst_index.qmd
+++ b/tests/tst_index.qmd
@@ -1,20 +1,22 @@
---
-title: "HAMUX QMD"
+title: Example Project (from YAML)
---
-# HAMUX QMD
-> Energy formulation for deep learning
+
+# Example Project
+> Example project
+
```{python}
#| echo: false
-# from hamux_qmd.core import *
+# from example_proj.core import *
```
-This is an insertion of another bit of markdown.
+With an executable print example:
-``` {python .code-cell-2}
-#| echo: true
-print(6+7)
+```{python}
+#| echo: true
+print(3+4)
```
This file will become your README and also the index of your documentation.
@@ -23,16 +25,16 @@ This file will become your README and also the index of your documentation.
If you are new to using `nbdev` here are some useful pointers to get you started.
-### Install hamux_qmd in Development mode
+### Install example_proj in Development mode
```sh
-# make sure hamux_qmd package is installed in development mode
+# make sure example_proj package is installed in development mode
$ pip install -e .
# make changes under nbs/ directory
# ...
-# compile to have changes apply to hamux_qmd
+# compile to have changes apply to example_proj
$ nbdev_prepare
```
@@ -43,32 +45,27 @@ $ nbdev_prepare
Install latest from the GitHub [repository][repo]:
```sh
-$ pip install git+https://github.com/bhoov/hamux_qmd.git
+$ pip install ...
```
or from [conda][conda]
```sh
-$ conda install -c bhoov hamux_qmd
+$ conda install -c ...
```
or from [pypi][pypi]
```sh
-$ pip install hamux_qmd
+$ pip install ...
```
-
-[repo]: https://github.com/bhoov/hamux_qmd
-[docs]: https://bhoov.github.io/hamux_qmd/
-[pypi]: https://pypi.org/project/hamux_qmd/
-[conda]: https://anaconda.org/bhoov/hamux_qmd
-
## How to use
Fill me in please! Don't forget code examples:
+
```{python}
1+1
```
\ No newline at end of file
From 61092276bb6be6a68a32d187492a69c2a4ebe7d8 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Fri, 30 May 2025 15:51:26 -0400
Subject: [PATCH 10/31] Cleanup debugging statements
---
nbdev/serve_drv.py | 1 -
nbs/api/14_quarto.ipynb | 2 --
2 files changed, 3 deletions(-)
diff --git a/nbdev/serve_drv.py b/nbdev/serve_drv.py
index f336d86d6..9d77d7f2d 100644
--- a/nbdev/serve_drv.py
+++ b/nbdev/serve_drv.py
@@ -17,7 +17,6 @@ def exec_scr(src, dst, md):
dst.write_text(res + f.getvalue())
def exec_qmd(src, dst, cb):
- print(f"Executing QMD-derived notebook: {src}")
nb = read_qmd(src)
k = CaptureShell()
with working_directory(src.parent): k.run_all(nb, exc_stop=False, preproc=no_eval)
diff --git a/nbs/api/14_quarto.ipynb b/nbs/api/14_quarto.ipynb
index b7b0ad8c6..b15216807 100644
--- a/nbs/api/14_quarto.ipynb
+++ b/nbs/api/14_quarto.ipynb
@@ -235,7 +235,6 @@
" for subdir in drel.parts:\n",
" _dir = _dir.setdefault(subdir, dict())\n",
" if Path(name).suffix == '.qmd': name = Path(name).with_suffix('.ipynb') # .qmd files are converted to .ipynb before docs are rendered\n",
- " print(\"new name: \", name)\n",
" _dir[name] = str(name)\n",
"\n",
" _recursive_parser(dir_struct, _contents, Path())\n",
@@ -370,7 +369,6 @@
" nbdev.doclinks._build_modidx()\n",
" nbdev_sidebar.__wrapped__(path=path, **kwargs)\n",
" cache = proc_nbs(path, n_workers=n_workers, **kwargs)\n",
- " print(\"cache: \", cache)\n",
" return cache,cfg,path"
]
},
From 8a321809056d7242df0a37edf1a9ac2744a24174 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Fri, 30 May 2025 16:46:33 -0400
Subject: [PATCH 11/31] Enable back-syncing qmd from changes in .py files
---
nbdev/_modidx.py | 5 ++-
nbdev/process.py | 54 ++++++++++++++++++++++++--------
nbdev/quarto.py | 2 --
nbdev/sync.py | 2 +-
nbs/api/03_process.ipynb | 66 ++++++++++++++++++++++++++++++++--------
nbs/api/06_sync.ipynb | 46 +++++++++++++++++-----------
tests/tst_index.qmd | 2 +-
7 files changed, 130 insertions(+), 47 deletions(-)
diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py
index 92c97137c..e4588485a 100644
--- a/nbdev/_modidx.py
+++ b/nbdev/_modidx.py
@@ -175,6 +175,7 @@
'nbdev.process._directive': ('api/process.html#_directive', 'nbdev/process.py'),
'nbdev.process._is_direc': ('api/process.html#_is_direc', 'nbdev/process.py'),
'nbdev.process._mk_procs': ('api/process.html#_mk_procs', 'nbdev/process.py'),
+ 'nbdev.process._nb_to_qmd_str': ('api/process.html#_nb_to_qmd_str', 'nbdev/process.py'),
'nbdev.process._norm_quarto': ('api/process.html#_norm_quarto', 'nbdev/process.py'),
'nbdev.process._partition_cell': ('api/process.html#_partition_cell', 'nbdev/process.py'),
'nbdev.process._qmd_to_raw_cell': ('api/process.html#_qmd_to_raw_cell', 'nbdev/process.py'),
@@ -185,7 +186,9 @@
'nbdev.process.nb_lang': ('api/process.html#nb_lang', 'nbdev/process.py'),
'nbdev.process.opt_set': ('api/process.html#opt_set', 'nbdev/process.py'),
'nbdev.process.read_nb_or_qmd': ('api/process.html#read_nb_or_qmd', 'nbdev/process.py'),
- 'nbdev.process.read_qmd': ('api/process.html#read_qmd', 'nbdev/process.py')},
+ 'nbdev.process.read_qmd': ('api/process.html#read_qmd', 'nbdev/process.py'),
+ 'nbdev.process.write_nb_or_qmd': ('api/process.html#write_nb_or_qmd', 'nbdev/process.py'),
+ 'nbdev.process.write_qmd': ('api/process.html#write_qmd', 'nbdev/process.py')},
'nbdev.processors': { 'nbdev.processors.FilterDefaults': ('api/processors.html#filterdefaults', 'nbdev/processors.py'),
'nbdev.processors.FilterDefaults.__call__': ( 'api/processors.html#filterdefaults.__call__',
'nbdev/processors.py'),
diff --git a/nbdev/process.py b/nbdev/process.py
index b5bfb8b7a..27cd95327 100644
--- a/nbdev/process.py
+++ b/nbdev/process.py
@@ -3,8 +3,8 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/03_process.ipynb.
# %% auto 0
-__all__ = ['langs', 'read_qmd', 'nb_lang', 'first_code_ln', 'extract_directives', 'opt_set', 'instantiate', 'read_nb_or_qmd',
- 'NBProcessor', 'Processor']
+__all__ = ['langs', 'read_qmd', 'write_qmd', 'write_nb_or_qmd', 'nb_lang', 'first_code_ln', 'extract_directives', 'opt_set',
+ 'instantiate', 'read_nb_or_qmd', 'NBProcessor', 'Processor']
# %% ../nbs/api/03_process.ipynb
from .config import *
@@ -18,33 +18,37 @@
from collections import defaultdict
# %% ../nbs/api/03_process.ipynb
-def _qmd_to_raw_cell(source_str, cell_type_str):
+def _qmd_to_raw_cell(source_str, cell_type_str, qmd_metadata=None):
"""Create a default ipynb json cell"""
cell = {'cell_type': cell_type_str, 'metadata': {}, 'source': source_str}
if cell_type_str == 'code':
cell['execution_count'] = None
cell['outputs'] = []
+ if qmd_metadata: cell['qmd_metadata'] = qmd_metadata
return cell
-def read_qmd(path): # Renamed to clarify output
+def read_qmd(path):
"""Reads a .qmd file as an nb compatible with the rest of execnb and nbdev"""
content = Path(path).read_text(encoding='utf-8')
- cell_pat = re.compile(r"^(`{3,})\s*\{python[^\n]*\}\s*(.*?)^\1\s*$", re.MULTILINE | re.DOTALL)
+ # Modified regex to capture the metadata between {python and }
+ cell_pat = re.compile(r"^(`{3,})\s*\{python([^\}]*)\}\s*\n(.*?)^\1\s*$", re.MULTILINE | re.DOTALL)
- # `parts` will be [md_chunk, captured_backticks_1, captured_code_1, md_chunk_2, ...]
- # We just care about md and code chunks
+ # `parts` will be [md_chunk, captured_backticks_1, captured_metadata_1, captured_code_1, md_chunk_2, ...]
parts = cell_pat.split(content)
raw_cells = []
# Handle the first markdown segment (before any code cells, or all content if no code cells)
initial_md_source = parts[0].strip()
if initial_md_source: raw_cells.append(_qmd_to_raw_cell(initial_md_source, 'markdown'))
- for i in range(1, len(parts), 3):
- if i + 1 < len(parts):
- code_source = parts[i+1].strip()
- if code_source: raw_cells.append(_qmd_to_raw_cell(code_source, 'code'))
- if i + 2 < len(parts):
- intermediate_md_source = parts[i+2].strip()
+
+ # 4 items per match: [md, backticks, metadata?, code]
+ for i in range(1, len(parts), 4):
+ if i + 2 < len(parts): # We have backticks, metadata, and code
+ metadata = parts[i+1] # The captured metadata
+ code_source = parts[i+2].strip() # The captured code
+ if code_source: raw_cells.append(_qmd_to_raw_cell(code_source, 'code', metadata if metadata else None))
+ if i + 3 < len(parts): # Intermediate markdown
+ intermediate_md_source = parts[i+3].strip()
if intermediate_md_source: raw_cells.append(_qmd_to_raw_cell(intermediate_md_source, 'markdown'))
# Construct the final notebook dictionary
@@ -62,6 +66,30 @@ def read_qmd(path): # Renamed to clarify output
return dict2nb(notebook_dict)
+
+# %% ../nbs/api/03_process.ipynb
+def _nb_to_qmd_str(nb):
+ """Convert a notebook to a string in .qmd format"""
+ def cell_to_qmd(cell):
+ source = cell.source.rstrip('\n')
+ if cell.cell_type in ['markdown', 'raw']: return source
+ elif cell.cell_type == 'code':
+ qmd_metadata = getattr(cell, 'qmd_metadata', None)
+ if qmd_metadata: return f'```{{python{qmd_metadata}}}\n{source}\n```'
+ else: return f'```{{python}}\n{source}\n```'
+ return ''
+ return '\n\n'.join(filter(None, [cell_to_qmd(cell) for cell in nb.cells]))
+
+def write_qmd(nb, path):
+ """Write a notebook back to .qmd format"""
+ qmd_str = _nb_to_qmd_str(nb)
+ Path(path).write_text(qmd_str, encoding='utf-8')
+
+def write_nb_or_qmd(nb, path):
+ if Path(path).suffix == '.qmd': write_qmd(nb, path)
+ else: write_nb(nb, path)
+
+
# %% ../nbs/api/03_process.ipynb
# from https://github.com/quarto-dev/quarto-cli/blob/main/src/resources/jupyter/notebook.py
langs = defaultdict(
diff --git a/nbdev/quarto.py b/nbdev/quarto.py
index 968431c30..4c883ad55 100644
--- a/nbdev/quarto.py
+++ b/nbdev/quarto.py
@@ -131,7 +131,6 @@ def _f(a,b): return Path(a),b
for subdir in drel.parts:
_dir = _dir.setdefault(subdir, dict())
if Path(name).suffix == '.qmd': name = Path(name).with_suffix('.ipynb') # .qmd files are converted to .ipynb before docs are rendered
- print("new name: ", name)
_dir[name] = str(name)
_recursive_parser(dir_struct, _contents, Path())
@@ -208,7 +207,6 @@ def _pre_docs(path=None, n_workers:int=defaults.cpus, **kwargs):
nbdev.doclinks._build_modidx()
nbdev_sidebar.__wrapped__(path=path, **kwargs)
cache = proc_nbs(path, n_workers=n_workers, **kwargs)
- print("cache: ", cache)
return cache,cfg,path
# %% ../nbs/api/14_quarto.ipynb
diff --git a/nbdev/sync.py b/nbdev/sync.py
index c76a62bfa..68b0d6b44 100644
--- a/nbdev/sync.py
+++ b/nbdev/sync.py
@@ -57,7 +57,7 @@ def _update_nb(nb_path, cells, lib_dir):
nbcell = nbp.nb.cells[cell.idx]
dirs,_ = _partition_cell(nbcell, 'python')
nbcell.source = ''.join(dirs) + _to_absolute(cell.code, cell.py_path, lib_dir)
- write_nb(nbp.nb, nb_path)
+ write_nb_or_qmd(nbp.nb, nb_path)
# %% ../nbs/api/06_sync.ipynb
def _update_mod(py_path, lib_dir):
diff --git a/nbs/api/03_process.ipynb b/nbs/api/03_process.ipynb
index 1ec668d32..35e859626 100644
--- a/nbs/api/03_process.ipynb
+++ b/nbs/api/03_process.ipynb
@@ -80,33 +80,37 @@
"outputs": [],
"source": [
"#|export\n",
- "def _qmd_to_raw_cell(source_str, cell_type_str):\n",
+ "def _qmd_to_raw_cell(source_str, cell_type_str, qmd_metadata=None):\n",
" \"\"\"Create a default ipynb json cell\"\"\"\n",
" cell = {'cell_type': cell_type_str, 'metadata': {}, 'source': source_str}\n",
" if cell_type_str == 'code':\n",
" cell['execution_count'] = None\n",
" cell['outputs'] = []\n",
+ " if qmd_metadata: cell['qmd_metadata'] = qmd_metadata\n",
" return cell\n",
"\n",
- "def read_qmd(path): # Renamed to clarify output\n",
+ "def read_qmd(path): \n",
" \"\"\"Reads a .qmd file as an nb compatible with the rest of execnb and nbdev\"\"\"\n",
" content = Path(path).read_text(encoding='utf-8')\n",
- " cell_pat = re.compile(r\"^(`{3,})\\s*\\{python[^\\n]*\\}\\s*(.*?)^\\1\\s*$\", re.MULTILINE | re.DOTALL)\n",
+ " # Modified regex to capture the metadata between {python and }\n",
+ " cell_pat = re.compile(r\"^(`{3,})\\s*\\{python([^\\}]*)\\}\\s*\\n(.*?)^\\1\\s*$\", re.MULTILINE | re.DOTALL)\n",
" \n",
- " # `parts` will be [md_chunk, captured_backticks_1, captured_code_1, md_chunk_2, ...]\n",
- " # We just care about md and code chunks\n",
+ " # `parts` will be [md_chunk, captured_backticks_1, captured_metadata_1, captured_code_1, md_chunk_2, ...]\n",
" parts = cell_pat.split(content)\n",
" raw_cells = []\n",
" \n",
" # Handle the first markdown segment (before any code cells, or all content if no code cells)\n",
" initial_md_source = parts[0].strip()\n",
" if initial_md_source: raw_cells.append(_qmd_to_raw_cell(initial_md_source, 'markdown'))\n",
- " for i in range(1, len(parts), 3):\n",
- " if i + 1 < len(parts):\n",
- " code_source = parts[i+1].strip()\n",
- " if code_source: raw_cells.append(_qmd_to_raw_cell(code_source, 'code'))\n",
- " if i + 2 < len(parts):\n",
- " intermediate_md_source = parts[i+2].strip()\n",
+ " \n",
+ " # 4 items per match: [md, backticks, metadata?, code]\n",
+ " for i in range(1, len(parts), 4):\n",
+ " if i + 2 < len(parts): # We have backticks, metadata, and code\n",
+ " metadata = parts[i+1] # The captured metadata\n",
+ " code_source = parts[i+2].strip() # The captured code\n",
+ " if code_source: raw_cells.append(_qmd_to_raw_cell(code_source, 'code', metadata if metadata else None))\n",
+ " if i + 3 < len(parts): # Intermediate markdown\n",
+ " intermediate_md_source = parts[i+3].strip()\n",
" if intermediate_md_source: raw_cells.append(_qmd_to_raw_cell(intermediate_md_source, 'markdown'))\n",
" \n",
" # Construct the final notebook dictionary\n",
@@ -122,7 +126,7 @@
" 'path_': str(path)\n",
" }\n",
" \n",
- " return dict2nb(notebook_dict)"
+ " return dict2nb(notebook_dict)\n"
]
},
{
@@ -194,6 +198,44 @@
"for match in matches: print(match)\n"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "a992cbfd",
+ "metadata": {},
+ "source": [
+ "We similarly need a `write_qmd` function to write a notebook to a .qmd file."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "07401f41",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| export\n",
+ "def _nb_to_qmd_str(nb):\n",
+ " \"\"\"Convert a notebook to a string in .qmd format\"\"\"\n",
+ " def cell_to_qmd(cell):\n",
+ " source = cell.source.rstrip('\\n')\n",
+ " if cell.cell_type in ['markdown', 'raw']: return source\n",
+ " elif cell.cell_type == 'code':\n",
+ " qmd_metadata = getattr(cell, 'qmd_metadata', None)\n",
+ " if qmd_metadata: return f'```{{python{qmd_metadata}}}\\n{source}\\n```'\n",
+ " else: return f'```{{python}}\\n{source}\\n```'\n",
+ " return ''\n",
+ " return '\\n\\n'.join(filter(None, [cell_to_qmd(cell) for cell in nb.cells]))\n",
+ "\n",
+ "def write_qmd(nb, path):\n",
+ " \"\"\"Write a notebook back to .qmd format\"\"\"\n",
+ " qmd_str = _nb_to_qmd_str(nb)\n",
+ " Path(path).write_text(qmd_str, encoding='utf-8')\n",
+ " \n",
+ "def write_nb_or_qmd(nb, path):\n",
+ " if Path(path).suffix == '.qmd': write_qmd(nb, path)\n",
+ " else: write_nb(nb, path)\n"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
diff --git a/nbs/api/06_sync.ipynb b/nbs/api/06_sync.ipynb
index 23dbcb33f..81d6e4ae3 100644
--- a/nbs/api/06_sync.ipynb
+++ b/nbs/api/06_sync.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
@@ -28,7 +28,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
@@ -51,7 +51,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
@@ -61,7 +61,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
@@ -76,7 +76,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
@@ -91,7 +91,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
@@ -107,7 +107,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
@@ -117,7 +117,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
@@ -128,7 +128,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
@@ -141,7 +141,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 25,
"metadata": {},
"outputs": [],
"source": [
@@ -155,12 +155,12 @@
" nbcell = nbp.nb.cells[cell.idx]\n",
" dirs,_ = _partition_cell(nbcell, 'python')\n",
" nbcell.source = ''.join(dirs) + _to_absolute(cell.code, cell.py_path, lib_dir)\n",
- " write_nb(nbp.nb, nb_path)"
+ " write_nb_or_qmd(nbp.nb, nb_path)"
]
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 26,
"metadata": {},
"outputs": [],
"source": [
@@ -173,7 +173,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
@@ -183,7 +183,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
@@ -203,7 +203,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
@@ -220,7 +220,7 @@
},
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 30,
"metadata": {},
"outputs": [],
"source": [
@@ -240,9 +240,21 @@
],
"metadata": {
"kernelspec": {
- "display_name": "python3",
+ "display_name": "nbdev",
"language": "python",
"name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.16"
}
},
"nbformat": 4,
diff --git a/tests/tst_index.qmd b/tests/tst_index.qmd
index c3ee1cad2..42b00501a 100644
--- a/tests/tst_index.qmd
+++ b/tests/tst_index.qmd
@@ -14,7 +14,7 @@ title: Example Project (from YAML)
With an executable print example:
-```{python}
+```{python .metadata thiskey=islost}
#| echo: true
print(3+4)
```
From 98f118d95c71745f974cc2e449b984d4950d9acc Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Fri, 30 May 2025 21:56:45 -0400
Subject: [PATCH 12/31] Only use official frontmatter for .qmd documents
---
nbdev/frontmatter.py | 6 ++++--
nbs/api/09_frontmatter.ipynb | 6 ++++--
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/nbdev/frontmatter.py b/nbdev/frontmatter.py
index 075322219..2c731b747 100644
--- a/nbdev/frontmatter.py
+++ b/nbdev/frontmatter.py
@@ -48,7 +48,9 @@ def _insertfm(nb, fm): nb.cells.insert(0, mk_cell(_dict2fm(fm), 'raw'))
class FrontmatterProc(Processor):
"A YAML and formatted-markdown frontmatter processor"
- def begin(self): self.fm = getattr(self.nb, 'frontmatter_', {})
+ def begin(self):
+ self.fm = getattr(self.nb, 'frontmatter_', {})
+ self.is_qmd = hasattr(self.nb, 'path_') and Path(self.nb.path_).suffix == '.qmd'
def _update(self, f, cell):
s = cell.get('source')
@@ -60,7 +62,7 @@ def _update(self, f, cell):
def cell(self, cell):
if cell.cell_type=='raw': self._update(_fm2dict, cell)
- elif cell.cell_type=='markdown' and 'title' not in self.fm: self._update(_md2dict, cell)
+ elif (cell.cell_type=='markdown' and 'title' not in self.fm and not self.is_qmd): self._update(_md2dict, cell)
def end(self):
self.nb.frontmatter_ = self.fm
diff --git a/nbs/api/09_frontmatter.ipynb b/nbs/api/09_frontmatter.ipynb
index aaec3d943..10c22c382 100644
--- a/nbs/api/09_frontmatter.ipynb
+++ b/nbs/api/09_frontmatter.ipynb
@@ -109,7 +109,9 @@
"\n",
"class FrontmatterProc(Processor):\n",
" \"A YAML and formatted-markdown frontmatter processor\"\n",
- " def begin(self): self.fm = getattr(self.nb, 'frontmatter_', {})\n",
+ " def begin(self): \n",
+ " self.fm = getattr(self.nb, 'frontmatter_', {})\n",
+ " self.is_qmd = hasattr(self.nb, 'path_') and Path(self.nb.path_).suffix == '.qmd'\n",
"\n",
" def _update(self, f, cell):\n",
" s = cell.get('source')\n",
@@ -121,7 +123,7 @@
"\n",
" def cell(self, cell):\n",
" if cell.cell_type=='raw': self._update(_fm2dict, cell)\n",
- " elif cell.cell_type=='markdown' and 'title' not in self.fm: self._update(_md2dict, cell)\n",
+ " elif (cell.cell_type=='markdown' and 'title' not in self.fm and not self.is_qmd): self._update(_md2dict, cell)\n",
"\n",
" def end(self):\n",
" self.nb.frontmatter_ = self.fm\n",
From 8b98a618bcf8622be047efa0c075a02f2b4d7a06 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Fri, 30 May 2025 22:01:59 -0400
Subject: [PATCH 13/31] Add documentation for no custom YAML for qmd
---
nbs/api/09_frontmatter.ipynb | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/nbs/api/09_frontmatter.ipynb b/nbs/api/09_frontmatter.ipynb
index 10c22c382..ee09663e4 100644
--- a/nbs/api/09_frontmatter.ipynb
+++ b/nbs/api/09_frontmatter.ipynb
@@ -145,7 +145,7 @@
"YAML frontmatter can be added to notebooks in one of two ways:\n",
"\n",
"1. By adding a raw notebook cell with `---` as the first and last lines, and YAML between them, or\n",
- "2. A specially formatted markdown cell. The first line should be start with a single `#` (creating an H1 heading), and becomes the title. Then, optionally, a line beginning with `>` (creating a quote block), which becomes the description. Finally, zero or more lines beginning with `- ` (creating a list), each of which contains YAML. (If you already have \"title\" defined in frontmatter in a raw cell, then markdown cells will be ignored.)\n",
+ "2. (**only for `.ipynb` files**) A specially formatted markdown cell. The first line should be start with a single `#` (creating an H1 heading), and becomes the title. Then, optionally, a line beginning with `>` (creating a quote block), which becomes the description. Finally, zero or more lines beginning with `- ` (creating a list), each of which contains YAML. (If you already have \"title\" defined in frontmatter in a raw cell, then markdown cells will be ignored.)\n",
"\n",
"For instance, our test notebook contains the following markdown cell:\n",
"\n",
From d43cf36bc815b8a0960681f66718cf111be11b8f Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Fri, 30 May 2025 22:11:50 -0400
Subject: [PATCH 14/31] Move qmd processing to 15_qmd.ipynb
---
nbdev/_modidx.py | 16 ++--
nbdev/process.py | 82 +----------------
nbdev/qmd.py | 84 ++++++++++++++++-
nbs/api/03_process.ipynb | 188 +--------------------------------------
nbs/api/06_sync.ipynb | 44 ++++-----
nbs/api/15_qmd.ipynb | 178 +++++++++++++++++++++++++++++++++++-
6 files changed, 286 insertions(+), 306 deletions(-)
diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py
index e4588485a..5f282a13f 100644
--- a/nbdev/_modidx.py
+++ b/nbdev/_modidx.py
@@ -175,20 +175,14 @@
'nbdev.process._directive': ('api/process.html#_directive', 'nbdev/process.py'),
'nbdev.process._is_direc': ('api/process.html#_is_direc', 'nbdev/process.py'),
'nbdev.process._mk_procs': ('api/process.html#_mk_procs', 'nbdev/process.py'),
- 'nbdev.process._nb_to_qmd_str': ('api/process.html#_nb_to_qmd_str', 'nbdev/process.py'),
'nbdev.process._norm_quarto': ('api/process.html#_norm_quarto', 'nbdev/process.py'),
'nbdev.process._partition_cell': ('api/process.html#_partition_cell', 'nbdev/process.py'),
- 'nbdev.process._qmd_to_raw_cell': ('api/process.html#_qmd_to_raw_cell', 'nbdev/process.py'),
'nbdev.process._quarto_re': ('api/process.html#_quarto_re', 'nbdev/process.py'),
'nbdev.process.extract_directives': ('api/process.html#extract_directives', 'nbdev/process.py'),
'nbdev.process.first_code_ln': ('api/process.html#first_code_ln', 'nbdev/process.py'),
'nbdev.process.instantiate': ('api/process.html#instantiate', 'nbdev/process.py'),
'nbdev.process.nb_lang': ('api/process.html#nb_lang', 'nbdev/process.py'),
- 'nbdev.process.opt_set': ('api/process.html#opt_set', 'nbdev/process.py'),
- 'nbdev.process.read_nb_or_qmd': ('api/process.html#read_nb_or_qmd', 'nbdev/process.py'),
- 'nbdev.process.read_qmd': ('api/process.html#read_qmd', 'nbdev/process.py'),
- 'nbdev.process.write_nb_or_qmd': ('api/process.html#write_nb_or_qmd', 'nbdev/process.py'),
- 'nbdev.process.write_qmd': ('api/process.html#write_qmd', 'nbdev/process.py')},
+ 'nbdev.process.opt_set': ('api/process.html#opt_set', 'nbdev/process.py')},
'nbdev.processors': { 'nbdev.processors.FilterDefaults': ('api/processors.html#filterdefaults', 'nbdev/processors.py'),
'nbdev.processors.FilterDefaults.__call__': ( 'api/processors.html#filterdefaults.__call__',
'nbdev/processors.py'),
@@ -247,12 +241,18 @@
'nbdev.processors.strip_hidden_metadata': ( 'api/processors.html#strip_hidden_metadata',
'nbdev/processors.py')},
'nbdev.qmd': { 'nbdev.qmd._install_nbdev': ('api/qmd.html#_install_nbdev', 'nbdev/qmd.py'),
+ 'nbdev.qmd._nb_to_qmd_str': ('api/qmd.html#_nb_to_qmd_str', 'nbdev/qmd.py'),
+ 'nbdev.qmd._qmd_to_raw_cell': ('api/qmd.html#_qmd_to_raw_cell', 'nbdev/qmd.py'),
'nbdev.qmd.btn': ('api/qmd.html#btn', 'nbdev/qmd.py'),
'nbdev.qmd.div': ('api/qmd.html#div', 'nbdev/qmd.py'),
'nbdev.qmd.img': ('api/qmd.html#img', 'nbdev/qmd.py'),
'nbdev.qmd.meta': ('api/qmd.html#meta', 'nbdev/qmd.py'),
+ 'nbdev.qmd.read_nb_or_qmd': ('api/qmd.html#read_nb_or_qmd', 'nbdev/qmd.py'),
+ 'nbdev.qmd.read_qmd': ('api/qmd.html#read_qmd', 'nbdev/qmd.py'),
'nbdev.qmd.tbl_row': ('api/qmd.html#tbl_row', 'nbdev/qmd.py'),
- 'nbdev.qmd.tbl_sep': ('api/qmd.html#tbl_sep', 'nbdev/qmd.py')},
+ 'nbdev.qmd.tbl_sep': ('api/qmd.html#tbl_sep', 'nbdev/qmd.py'),
+ 'nbdev.qmd.write_nb_or_qmd': ('api/qmd.html#write_nb_or_qmd', 'nbdev/qmd.py'),
+ 'nbdev.qmd.write_qmd': ('api/qmd.html#write_qmd', 'nbdev/qmd.py')},
'nbdev.quarto': { 'nbdev.quarto.IndentDumper': ('api/quarto.html#indentdumper', 'nbdev/quarto.py'),
'nbdev.quarto.IndentDumper.increase_indent': ( 'api/quarto.html#indentdumper.increase_indent',
'nbdev/quarto.py'),
diff --git a/nbdev/process.py b/nbdev/process.py
index 27cd95327..7a526e519 100644
--- a/nbdev/process.py
+++ b/nbdev/process.py
@@ -3,13 +3,13 @@
# AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/api/03_process.ipynb.
# %% auto 0
-__all__ = ['langs', 'read_qmd', 'write_qmd', 'write_nb_or_qmd', 'nb_lang', 'first_code_ln', 'extract_directives', 'opt_set',
- 'instantiate', 'read_nb_or_qmd', 'NBProcessor', 'Processor']
+__all__ = ['langs', 'nb_lang', 'first_code_ln', 'extract_directives', 'opt_set', 'instantiate', 'NBProcessor', 'Processor']
# %% ../nbs/api/03_process.ipynb
from .config import *
from .maker import *
from .imports import *
+from .qmd import *
from execnb.nbio import *
from fastcore.script import *
@@ -17,79 +17,6 @@
from collections import defaultdict
-# %% ../nbs/api/03_process.ipynb
-def _qmd_to_raw_cell(source_str, cell_type_str, qmd_metadata=None):
- """Create a default ipynb json cell"""
- cell = {'cell_type': cell_type_str, 'metadata': {}, 'source': source_str}
- if cell_type_str == 'code':
- cell['execution_count'] = None
- cell['outputs'] = []
- if qmd_metadata: cell['qmd_metadata'] = qmd_metadata
- return cell
-
-def read_qmd(path):
- """Reads a .qmd file as an nb compatible with the rest of execnb and nbdev"""
- content = Path(path).read_text(encoding='utf-8')
- # Modified regex to capture the metadata between {python and }
- cell_pat = re.compile(r"^(`{3,})\s*\{python([^\}]*)\}\s*\n(.*?)^\1\s*$", re.MULTILINE | re.DOTALL)
-
- # `parts` will be [md_chunk, captured_backticks_1, captured_metadata_1, captured_code_1, md_chunk_2, ...]
- parts = cell_pat.split(content)
- raw_cells = []
-
- # Handle the first markdown segment (before any code cells, or all content if no code cells)
- initial_md_source = parts[0].strip()
- if initial_md_source: raw_cells.append(_qmd_to_raw_cell(initial_md_source, 'markdown'))
-
- # 4 items per match: [md, backticks, metadata?, code]
- for i in range(1, len(parts), 4):
- if i + 2 < len(parts): # We have backticks, metadata, and code
- metadata = parts[i+1] # The captured metadata
- code_source = parts[i+2].strip() # The captured code
- if code_source: raw_cells.append(_qmd_to_raw_cell(code_source, 'code', metadata if metadata else None))
- if i + 3 < len(parts): # Intermediate markdown
- intermediate_md_source = parts[i+3].strip()
- if intermediate_md_source: raw_cells.append(_qmd_to_raw_cell(intermediate_md_source, 'markdown'))
-
- # Construct the final notebook dictionary
- notebook_dict = {
- 'cells': raw_cells,
- 'metadata': {
- 'kernelspec': {'display_name': 'Python 3', 'language': 'python', 'name': 'python3'},
- 'language_info': {'name': 'python'},
- 'path': str(path)
- },
- 'nbformat': 4,
- 'nbformat_minor': 5,
- 'path_': str(path)
- }
-
- return dict2nb(notebook_dict)
-
-
-# %% ../nbs/api/03_process.ipynb
-def _nb_to_qmd_str(nb):
- """Convert a notebook to a string in .qmd format"""
- def cell_to_qmd(cell):
- source = cell.source.rstrip('\n')
- if cell.cell_type in ['markdown', 'raw']: return source
- elif cell.cell_type == 'code':
- qmd_metadata = getattr(cell, 'qmd_metadata', None)
- if qmd_metadata: return f'```{{python{qmd_metadata}}}\n{source}\n```'
- else: return f'```{{python}}\n{source}\n```'
- return ''
- return '\n\n'.join(filter(None, [cell_to_qmd(cell) for cell in nb.cells]))
-
-def write_qmd(nb, path):
- """Write a notebook back to .qmd format"""
- qmd_str = _nb_to_qmd_str(nb)
- Path(path).write_text(qmd_str, encoding='utf-8')
-
-def write_nb_or_qmd(nb, path):
- if Path(path).suffix == '.qmd': write_qmd(nb, path)
- else: write_nb(nb, path)
-
-
# %% ../nbs/api/03_process.ipynb
# from https://github.com/quarto-dev/quarto-cli/blob/main/src/resources/jupyter/notebook.py
langs = defaultdict(
@@ -162,11 +89,6 @@ def _mk_procs(procs, nb): return L(procs).map(instantiate, nb=nb)
# %% ../nbs/api/03_process.ipynb
def _is_direc(f): return getattr(f, '__name__', '-')[-1]=='_'
-# %% ../nbs/api/03_process.ipynb
-def read_nb_or_qmd(path):
- if Path(path).suffix == '.qmd': return read_qmd(path)
- return read_nb(path)
-
# %% ../nbs/api/03_process.ipynb
class NBProcessor:
"Process cells and nbdev comments in a notebook"
diff --git a/nbdev/qmd.py b/nbdev/qmd.py
index 772895695..d3b69227b 100644
--- a/nbdev/qmd.py
+++ b/nbdev/qmd.py
@@ -4,13 +4,95 @@
# %% ../nbs/api/15_qmd.ipynb 2
from __future__ import annotations
+from .config import *
+from execnb.nbio import *
+from pathlib import Path
+import re
+
import sys,os,inspect
from fastcore.utils import *
from fastcore.meta import delegates
# %% auto 0
-__all__ = ['meta', 'div', 'img', 'btn', 'tbl_row', 'tbl_sep']
+__all__ = ['read_qmd', 'read_nb_or_qmd', 'write_qmd', 'write_nb_or_qmd', 'meta', 'div', 'img', 'btn', 'tbl_row', 'tbl_sep']
+
+# %% ../nbs/api/15_qmd.ipynb
+def _qmd_to_raw_cell(source_str, cell_type_str, qmd_metadata=None):
+ """Create a default ipynb json cell"""
+ cell = {'cell_type': cell_type_str, 'metadata': {}, 'source': source_str}
+ if cell_type_str == 'code':
+ cell['execution_count'] = None
+ cell['outputs'] = []
+ if qmd_metadata: cell['qmd_metadata'] = qmd_metadata
+ return cell
+
+def read_qmd(path):
+ """Reads a .qmd file as an nb compatible with the rest of execnb and nbdev"""
+ content = Path(path).read_text(encoding='utf-8')
+ # Modified regex to capture the metadata between {python and }
+ cell_pat = re.compile(r"^(`{3,})\s*\{python([^\}]*)\}\s*\n(.*?)^\1\s*$", re.MULTILINE | re.DOTALL)
+
+ # `parts` will be [md_chunk, captured_backticks_1, captured_metadata_1, captured_code_1, md_chunk_2, ...]
+ parts = cell_pat.split(content)
+ raw_cells = []
+
+ # Handle the first markdown segment (before any code cells, or all content if no code cells)
+ initial_md_source = parts[0].strip()
+ if initial_md_source: raw_cells.append(_qmd_to_raw_cell(initial_md_source, 'markdown'))
+
+ # 4 items per match: [md, backticks, metadata?, code]
+ for i in range(1, len(parts), 4):
+ if i + 2 < len(parts): # We have backticks, metadata, and code
+ metadata = parts[i+1] # The captured metadata
+ code_source = parts[i+2].strip() # The captured code
+ if code_source: raw_cells.append(_qmd_to_raw_cell(code_source, 'code', metadata if metadata else None))
+ if i + 3 < len(parts): # Intermediate markdown
+ intermediate_md_source = parts[i+3].strip()
+ if intermediate_md_source: raw_cells.append(_qmd_to_raw_cell(intermediate_md_source, 'markdown'))
+
+ # Construct the final notebook dictionary
+ notebook_dict = {
+ 'cells': raw_cells,
+ 'metadata': {
+ 'kernelspec': {'display_name': 'Python 3', 'language': 'python', 'name': 'python3'},
+ 'language_info': {'name': 'python'},
+ 'path': str(path)
+ },
+ 'nbformat': 4,
+ 'nbformat_minor': 5,
+ 'path_': str(path)
+ }
+
+ return dict2nb(notebook_dict)
+
+def read_nb_or_qmd(path):
+ if Path(path).suffix == '.qmd': return read_qmd(path)
+ return read_nb(path)
+
+
+# %% ../nbs/api/15_qmd.ipynb
+def _nb_to_qmd_str(nb):
+ """Convert a notebook to a string in .qmd format"""
+ def cell_to_qmd(cell):
+ source = cell.source.rstrip('\n')
+ if cell.cell_type in ['markdown', 'raw']: return source
+ elif cell.cell_type == 'code':
+ qmd_metadata = getattr(cell, 'qmd_metadata', None)
+ if qmd_metadata: return f'```{{python{qmd_metadata}}}\n{source}\n```'
+ else: return f'```{{python}}\n{source}\n```'
+ return ''
+ return '\n\n'.join(filter(None, [cell_to_qmd(cell) for cell in nb.cells]))
+
+def write_qmd(nb, path):
+ """Write a notebook back to .qmd format"""
+ qmd_str = _nb_to_qmd_str(nb)
+ Path(path).write_text(qmd_str, encoding='utf-8')
+
+def write_nb_or_qmd(nb, path):
+ if Path(path).suffix == '.qmd': write_qmd(nb, path)
+ else: write_nb(nb, path)
+
# %% ../nbs/api/15_qmd.ipynb
def meta(md, # Markdown to add meta to
diff --git a/nbs/api/03_process.ipynb b/nbs/api/03_process.ipynb
index 35e859626..c1c98ea96 100644
--- a/nbs/api/03_process.ipynb
+++ b/nbs/api/03_process.ipynb
@@ -32,6 +32,7 @@
"from nbdev.config import *\n",
"from nbdev.maker import *\n",
"from nbdev.imports import *\n",
+ "from nbdev.qmd import *\n",
"\n",
"from execnb.nbio import *\n",
"from fastcore.script import *\n",
@@ -72,170 +73,6 @@
"nb_minimal = read_nb('../../tests/minimal.ipynb')"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "ca351f3d",
- "metadata": {},
- "outputs": [],
- "source": [
- "#|export\n",
- "def _qmd_to_raw_cell(source_str, cell_type_str, qmd_metadata=None):\n",
- " \"\"\"Create a default ipynb json cell\"\"\"\n",
- " cell = {'cell_type': cell_type_str, 'metadata': {}, 'source': source_str}\n",
- " if cell_type_str == 'code':\n",
- " cell['execution_count'] = None\n",
- " cell['outputs'] = []\n",
- " if qmd_metadata: cell['qmd_metadata'] = qmd_metadata\n",
- " return cell\n",
- "\n",
- "def read_qmd(path): \n",
- " \"\"\"Reads a .qmd file as an nb compatible with the rest of execnb and nbdev\"\"\"\n",
- " content = Path(path).read_text(encoding='utf-8')\n",
- " # Modified regex to capture the metadata between {python and }\n",
- " cell_pat = re.compile(r\"^(`{3,})\\s*\\{python([^\\}]*)\\}\\s*\\n(.*?)^\\1\\s*$\", re.MULTILINE | re.DOTALL)\n",
- " \n",
- " # `parts` will be [md_chunk, captured_backticks_1, captured_metadata_1, captured_code_1, md_chunk_2, ...]\n",
- " parts = cell_pat.split(content)\n",
- " raw_cells = []\n",
- " \n",
- " # Handle the first markdown segment (before any code cells, or all content if no code cells)\n",
- " initial_md_source = parts[0].strip()\n",
- " if initial_md_source: raw_cells.append(_qmd_to_raw_cell(initial_md_source, 'markdown'))\n",
- " \n",
- " # 4 items per match: [md, backticks, metadata?, code]\n",
- " for i in range(1, len(parts), 4):\n",
- " if i + 2 < len(parts): # We have backticks, metadata, and code\n",
- " metadata = parts[i+1] # The captured metadata\n",
- " code_source = parts[i+2].strip() # The captured code\n",
- " if code_source: raw_cells.append(_qmd_to_raw_cell(code_source, 'code', metadata if metadata else None))\n",
- " if i + 3 < len(parts): # Intermediate markdown\n",
- " intermediate_md_source = parts[i+3].strip()\n",
- " if intermediate_md_source: raw_cells.append(_qmd_to_raw_cell(intermediate_md_source, 'markdown'))\n",
- " \n",
- " # Construct the final notebook dictionary\n",
- " notebook_dict = {\n",
- " 'cells': raw_cells,\n",
- " 'metadata': {\n",
- " 'kernelspec': {'display_name': 'Python 3', 'language': 'python', 'name': 'python3'},\n",
- " 'language_info': {'name': 'python'},\n",
- " 'path': str(path)\n",
- " },\n",
- " 'nbformat': 4,\n",
- " 'nbformat_minor': 5,\n",
- " 'path_': str(path)\n",
- " }\n",
- " \n",
- " return dict2nb(notebook_dict)\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "1559503a",
- "metadata": {},
- "outputs": [],
- "source": [
- "#| hide\n",
- "# Test: we want the .qmd file to be very close to .ipynb file in the nb source (except for saved outputs which are not present in .qmd)\n",
- "nb_qmd = read_qmd(\"../../tests/tst_index.qmd\")\n",
- "nb_ipynb = read_nb(\"../../tests/tst_index.ipynb\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "d6c39f04",
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "('```', '#| export\\nprint(3+4)\\n')\n",
- "('````', '#| export\\nprint(9+12)\\n')\n",
- "('`````', 'print(5+6)\\n')\n"
- ]
- }
- ],
- "source": [
- "#| hide\n",
- "tst_cell = \"\"\"\\\n",
- "# Title \n",
- "> description\n",
- "\n",
- "and a couple more pieces of information before the code cell:\n",
- "\n",
- "``` {python .code-cell-2}\n",
- "#| export\n",
- "print(3+4)\n",
- "```\n",
- "\n",
- "```` {python .code-cell-2}\n",
- "#| export\n",
- "print(9+12)\n",
- "````\n",
- "\n",
- "`````{python}\n",
- "print(5+6)\n",
- "`````\n",
- "\n",
- "```python\n",
- "print(\"not a cell\")\n",
- "```\n",
- "\"\"\"\n",
- "cell_pat = re.compile(r\"^(`{3,})\\s*\\{python[^\\n]*\\}\\s*(.*?)^\\1\\s*$\", re.MULTILINE | re.DOTALL)\n",
- "matches = cell_pat.findall(tst_cell)\n",
- "assert len(matches) == 3\n",
- "assert len(matches[0][0]) == 3 # 3 backticks\n",
- "assert len(matches[1][0]) == 4 # 4 backticks\n",
- "assert len(matches[2][0]) == 5 # 5 backticks\n",
- "\n",
- "assert matches[0][1] == '#| export\\nprint(3+4)\\n'\n",
- "assert matches[1][1] == '#| export\\nprint(9+12)\\n'\n",
- "assert matches[2][1] == 'print(5+6)\\n'\n",
- "\n",
- "for match in matches: print(match)\n"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "a992cbfd",
- "metadata": {},
- "source": [
- "We similarly need a `write_qmd` function to write a notebook to a .qmd file."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "07401f41",
- "metadata": {},
- "outputs": [],
- "source": [
- "#| export\n",
- "def _nb_to_qmd_str(nb):\n",
- " \"\"\"Convert a notebook to a string in .qmd format\"\"\"\n",
- " def cell_to_qmd(cell):\n",
- " source = cell.source.rstrip('\\n')\n",
- " if cell.cell_type in ['markdown', 'raw']: return source\n",
- " elif cell.cell_type == 'code':\n",
- " qmd_metadata = getattr(cell, 'qmd_metadata', None)\n",
- " if qmd_metadata: return f'```{{python{qmd_metadata}}}\\n{source}\\n```'\n",
- " else: return f'```{{python}}\\n{source}\\n```'\n",
- " return ''\n",
- " return '\\n\\n'.join(filter(None, [cell_to_qmd(cell) for cell in nb.cells]))\n",
- "\n",
- "def write_qmd(nb, path):\n",
- " \"\"\"Write a notebook back to .qmd format\"\"\"\n",
- " qmd_str = _nb_to_qmd_str(nb)\n",
- " Path(path).write_text(qmd_str, encoding='utf-8')\n",
- " \n",
- "def write_nb_or_qmd(nb, path):\n",
- " if Path(path).suffix == '.qmd': write_qmd(nb, path)\n",
- " else: write_nb(nb, path)\n"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
@@ -517,29 +354,6 @@
"def _is_direc(f): return getattr(f, '__name__', '-')[-1]=='_'"
]
},
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "91a54352",
- "metadata": {},
- "outputs": [],
- "source": [
- "#|export\n",
- "def read_nb_or_qmd(path):\n",
- " if Path(path).suffix == '.qmd': return read_qmd(path)\n",
- " return read_nb(path)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "b613908f",
- "metadata": {},
- "outputs": [],
- "source": [
- "out = read_nb_or_qmd('../../tests/minimal.qmd')"
- ]
- },
{
"cell_type": "code",
"execution_count": null,
diff --git a/nbs/api/06_sync.ipynb b/nbs/api/06_sync.ipynb
index 81d6e4ae3..80738ac38 100644
--- a/nbs/api/06_sync.ipynb
+++ b/nbs/api/06_sync.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -28,7 +28,7 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -51,7 +51,7 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -61,7 +61,7 @@
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -76,7 +76,7 @@
},
{
"cell_type": "code",
- "execution_count": 20,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -91,7 +91,7 @@
},
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -107,7 +107,7 @@
},
{
"cell_type": "code",
- "execution_count": 22,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -117,7 +117,7 @@
},
{
"cell_type": "code",
- "execution_count": 23,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -128,7 +128,7 @@
},
{
"cell_type": "code",
- "execution_count": 24,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -141,7 +141,7 @@
},
{
"cell_type": "code",
- "execution_count": 25,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -160,7 +160,7 @@
},
{
"cell_type": "code",
- "execution_count": 26,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -173,7 +173,7 @@
},
{
"cell_type": "code",
- "execution_count": 27,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -183,7 +183,7 @@
},
{
"cell_type": "code",
- "execution_count": 28,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -203,7 +203,7 @@
},
{
"cell_type": "code",
- "execution_count": 29,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -220,7 +220,7 @@
},
{
"cell_type": "code",
- "execution_count": 30,
+ "execution_count": null,
"metadata": {},
"outputs": [],
"source": [
@@ -240,21 +240,9 @@
],
"metadata": {
"kernelspec": {
- "display_name": "nbdev",
+ "display_name": "python3",
"language": "python",
"name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.10.16"
}
},
"nbformat": 4,
diff --git a/nbs/api/15_qmd.ipynb b/nbs/api/15_qmd.ipynb
index b23df49d6..a2ee56bfd 100644
--- a/nbs/api/15_qmd.ipynb
+++ b/nbs/api/15_qmd.ipynb
@@ -18,18 +18,24 @@
"metadata": {},
"outputs": [],
"source": [
+ "#|hide\n",
"#|default_exp qmd"
]
},
{
"cell_type": "code",
"execution_count": null,
- "id": "6a35c7c4-748f-4c82-a9bf-c780a8d83e90",
+ "id": "1306d33a",
"metadata": {},
"outputs": [],
"source": [
"#|export\n",
"from __future__ import annotations\n",
+ "from nbdev.config import *\n",
+ "from execnb.nbio import *\n",
+ "from pathlib import Path\n",
+ "import re\n",
+ "\n",
"import sys,os,inspect\n",
"\n",
"from fastcore.utils import *\n",
@@ -39,7 +45,7 @@
{
"cell_type": "code",
"execution_count": null,
- "id": "1e623a3d-3e77-44c6-adf3-4768b78328c5",
+ "id": "481478f5",
"metadata": {},
"outputs": [],
"source": [
@@ -47,6 +53,174 @@
"from fastcore.test import *"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "19e7927a",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#|export\n",
+ "def _qmd_to_raw_cell(source_str, cell_type_str, qmd_metadata=None):\n",
+ " \"\"\"Create a default ipynb json cell\"\"\"\n",
+ " cell = {'cell_type': cell_type_str, 'metadata': {}, 'source': source_str}\n",
+ " if cell_type_str == 'code':\n",
+ " cell['execution_count'] = None\n",
+ " cell['outputs'] = []\n",
+ " if qmd_metadata: cell['qmd_metadata'] = qmd_metadata\n",
+ " return cell\n",
+ "\n",
+ "def read_qmd(path): \n",
+ " \"\"\"Reads a .qmd file as an nb compatible with the rest of execnb and nbdev\"\"\"\n",
+ " content = Path(path).read_text(encoding='utf-8')\n",
+ " # Modified regex to capture the metadata between {python and }\n",
+ " cell_pat = re.compile(r\"^(`{3,})\\s*\\{python([^\\}]*)\\}\\s*\\n(.*?)^\\1\\s*$\", re.MULTILINE | re.DOTALL)\n",
+ " \n",
+ " # `parts` will be [md_chunk, captured_backticks_1, captured_metadata_1, captured_code_1, md_chunk_2, ...]\n",
+ " parts = cell_pat.split(content)\n",
+ " raw_cells = []\n",
+ " \n",
+ " # Handle the first markdown segment (before any code cells, or all content if no code cells)\n",
+ " initial_md_source = parts[0].strip()\n",
+ " if initial_md_source: raw_cells.append(_qmd_to_raw_cell(initial_md_source, 'markdown'))\n",
+ " \n",
+ " # 4 items per match: [md, backticks, metadata?, code]\n",
+ " for i in range(1, len(parts), 4):\n",
+ " if i + 2 < len(parts): # We have backticks, metadata, and code\n",
+ " metadata = parts[i+1] # The captured metadata\n",
+ " code_source = parts[i+2].strip() # The captured code\n",
+ " if code_source: raw_cells.append(_qmd_to_raw_cell(code_source, 'code', metadata if metadata else None))\n",
+ " if i + 3 < len(parts): # Intermediate markdown\n",
+ " intermediate_md_source = parts[i+3].strip()\n",
+ " if intermediate_md_source: raw_cells.append(_qmd_to_raw_cell(intermediate_md_source, 'markdown'))\n",
+ " \n",
+ " # Construct the final notebook dictionary\n",
+ " notebook_dict = {\n",
+ " 'cells': raw_cells,\n",
+ " 'metadata': {\n",
+ " 'kernelspec': {'display_name': 'Python 3', 'language': 'python', 'name': 'python3'},\n",
+ " 'language_info': {'name': 'python'},\n",
+ " 'path': str(path)\n",
+ " },\n",
+ " 'nbformat': 4,\n",
+ " 'nbformat_minor': 5,\n",
+ " 'path_': str(path)\n",
+ " }\n",
+ " \n",
+ " return dict2nb(notebook_dict)\n",
+ "\n",
+ "def read_nb_or_qmd(path):\n",
+ " if Path(path).suffix == '.qmd': return read_qmd(path)\n",
+ " return read_nb(path)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1b9f86cd",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#| hide\n",
+ "# Test: we want the .qmd file to be very close to .ipynb file in the nb source (except for saved outputs which are not present in .qmd)\n",
+ "nb_qmd = read_qmd(\"../../tests/tst_index.qmd\")\n",
+ "nb_ipynb = read_nb(\"../../tests/tst_index.ipynb\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "24f36490",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "('```', '#| export\\nprint(3+4)\\n')\n",
+ "('````', '#| export\\nprint(9+12)\\n')\n",
+ "('`````', 'print(5+6)\\n')\n"
+ ]
+ }
+ ],
+ "source": [
+ "#| hide\n",
+ "tst_cell = \"\"\"\\\n",
+ "# Title \n",
+ "> description\n",
+ "\n",
+ "and a couple more pieces of information before the code cell:\n",
+ "\n",
+ "``` {python .code-cell-2}\n",
+ "#| export\n",
+ "print(3+4)\n",
+ "```\n",
+ "\n",
+ "```` {python .code-cell-2}\n",
+ "#| export\n",
+ "print(9+12)\n",
+ "````\n",
+ "\n",
+ "`````{python}\n",
+ "print(5+6)\n",
+ "`````\n",
+ "\n",
+ "```python\n",
+ "print(\"not a cell\")\n",
+ "```\n",
+ "\"\"\"\n",
+ "cell_pat = re.compile(r\"^(`{3,})\\s*\\{python[^\\n]*\\}\\s*(.*?)^\\1\\s*$\", re.MULTILINE | re.DOTALL)\n",
+ "matches = cell_pat.findall(tst_cell)\n",
+ "assert len(matches) == 3\n",
+ "assert len(matches[0][0]) == 3 # 3 backticks\n",
+ "assert len(matches[1][0]) == 4 # 4 backticks\n",
+ "assert len(matches[2][0]) == 5 # 5 backticks\n",
+ "\n",
+ "assert matches[0][1] == '#| export\\nprint(3+4)\\n'\n",
+ "assert matches[1][1] == '#| export\\nprint(9+12)\\n'\n",
+ "assert matches[2][1] == 'print(5+6)\\n'\n",
+ "\n",
+ "for match in matches: print(match)\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "efa97b75",
+ "metadata": {},
+ "source": [
+ "We similarly need a `write_qmd` function to write a notebook to a .qmd file."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "4c085a88",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#|export\n",
+ "def _nb_to_qmd_str(nb):\n",
+ " \"\"\"Convert a notebook to a string in .qmd format\"\"\"\n",
+ " def cell_to_qmd(cell):\n",
+ " source = cell.source.rstrip('\\n')\n",
+ " if cell.cell_type in ['markdown', 'raw']: return source\n",
+ " elif cell.cell_type == 'code':\n",
+ " qmd_metadata = getattr(cell, 'qmd_metadata', None)\n",
+ " if qmd_metadata: return f'```{{python{qmd_metadata}}}\\n{source}\\n```'\n",
+ " else: return f'```{{python}}\\n{source}\\n```'\n",
+ " return ''\n",
+ " return '\\n\\n'.join(filter(None, [cell_to_qmd(cell) for cell in nb.cells]))\n",
+ "\n",
+ "def write_qmd(nb, path):\n",
+ " \"\"\"Write a notebook back to .qmd format\"\"\"\n",
+ " qmd_str = _nb_to_qmd_str(nb)\n",
+ " Path(path).write_text(qmd_str, encoding='utf-8')\n",
+ " \n",
+ "def write_nb_or_qmd(nb, path):\n",
+ " if Path(path).suffix == '.qmd': write_qmd(nb, path)\n",
+ " else: write_nb(nb, path)\n"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
From adf7039f1f5e6c10687571bfb3689591fea7e0a9 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Fri, 30 May 2025 22:15:48 -0400
Subject: [PATCH 15/31] Allow .qmd frontmatter to still auto-calc title and
description, but not key: vals from lists
---
nbdev/frontmatter.py | 13 +++++++------
nbs/api/09_frontmatter.ipynb | 15 ++++++++-------
2 files changed, 15 insertions(+), 13 deletions(-)
diff --git a/nbdev/frontmatter.py b/nbdev/frontmatter.py
index 2c731b747..440522776 100644
--- a/nbdev/frontmatter.py
+++ b/nbdev/frontmatter.py
@@ -28,7 +28,7 @@ def _fm2dict(s:str, nb=True):
match = re_fm.search(s.strip())
return yaml.safe_load(match.group(1)) if match else {}
-def _md2dict(s:str):
+def _md2dict(s:str, find_kv=True):
"Convert H1 formatted markdown cell to frontmatter dict"
if '#' not in s: return {}
m = re.search(r'^#\s+(\S.*?)\s*$', s, flags=re.MULTILINE)
@@ -36,10 +36,11 @@ def _md2dict(s:str):
res = {'title': m.group(1)}
m = re.search(r'^>\s+(\S.*?)\s*$', s, flags=re.MULTILINE)
if m: res['description'] = m.group(1)
- r = re.findall(r'^-\s+(\S.*:.*\S)\s*$', s, flags=re.MULTILINE)
- if r:
- try: res.update(yaml.safe_load('\n'.join(r)))
- except Exception as e: warn(f'Failed to create YAML dict for:\n{r}\n\n{e}\n')
+ if find_kv:
+ r = re.findall(r'^-\s+(\S.*:.*\S)\s*$', s, flags=re.MULTILINE)
+ if r:
+ try: res.update(yaml.safe_load('\n'.join(r)))
+ except Exception as e: warn(f'Failed to create YAML dict for:\n{r}\n\n{e}\n')
return res
# %% ../nbs/api/09_frontmatter.ipynb
@@ -62,7 +63,7 @@ def _update(self, f, cell):
def cell(self, cell):
if cell.cell_type=='raw': self._update(_fm2dict, cell)
- elif (cell.cell_type=='markdown' and 'title' not in self.fm and not self.is_qmd): self._update(_md2dict, cell)
+ elif (cell.cell_type=='markdown' and 'title' not in self.fm): self._update(partial(_md2dict, find_kv=not self.is_qmd), cell)
def end(self):
self.nb.frontmatter_ = self.fm
diff --git a/nbs/api/09_frontmatter.ipynb b/nbs/api/09_frontmatter.ipynb
index ee09663e4..e59ac5cb3 100644
--- a/nbs/api/09_frontmatter.ipynb
+++ b/nbs/api/09_frontmatter.ipynb
@@ -81,7 +81,7 @@
" match = re_fm.search(s.strip())\n",
" return yaml.safe_load(match.group(1)) if match else {}\n",
"\n",
- "def _md2dict(s:str):\n",
+ "def _md2dict(s:str, find_kv=True):\n",
" \"Convert H1 formatted markdown cell to frontmatter dict\"\n",
" if '#' not in s: return {}\n",
" m = re.search(r'^#\\s+(\\S.*?)\\s*$', s, flags=re.MULTILINE)\n",
@@ -89,10 +89,11 @@
" res = {'title': m.group(1)}\n",
" m = re.search(r'^>\\s+(\\S.*?)\\s*$', s, flags=re.MULTILINE)\n",
" if m: res['description'] = m.group(1)\n",
- " r = re.findall(r'^-\\s+(\\S.*:.*\\S)\\s*$', s, flags=re.MULTILINE)\n",
- " if r:\n",
- " try: res.update(yaml.safe_load('\\n'.join(r)))\n",
- " except Exception as e: warn(f'Failed to create YAML dict for:\\n{r}\\n\\n{e}\\n')\n",
+ " if find_kv:\n",
+ " r = re.findall(r'^-\\s+(\\S.*:.*\\S)\\s*$', s, flags=re.MULTILINE)\n",
+ " if r:\n",
+ " try: res.update(yaml.safe_load('\\n'.join(r)))\n",
+ " except Exception as e: warn(f'Failed to create YAML dict for:\\n{r}\\n\\n{e}\\n')\n",
" return res"
]
},
@@ -123,7 +124,7 @@
"\n",
" def cell(self, cell):\n",
" if cell.cell_type=='raw': self._update(_fm2dict, cell)\n",
- " elif (cell.cell_type=='markdown' and 'title' not in self.fm and not self.is_qmd): self._update(_md2dict, cell)\n",
+ " elif (cell.cell_type=='markdown' and 'title' not in self.fm): self._update(partial(_md2dict, find_kv=not self.is_qmd), cell)\n",
"\n",
" def end(self):\n",
" self.nb.frontmatter_ = self.fm\n",
@@ -145,7 +146,7 @@
"YAML frontmatter can be added to notebooks in one of two ways:\n",
"\n",
"1. By adding a raw notebook cell with `---` as the first and last lines, and YAML between them, or\n",
- "2. (**only for `.ipynb` files**) A specially formatted markdown cell. The first line should be start with a single `#` (creating an H1 heading), and becomes the title. Then, optionally, a line beginning with `>` (creating a quote block), which becomes the description. Finally, zero or more lines beginning with `- ` (creating a list), each of which contains YAML. (If you already have \"title\" defined in frontmatter in a raw cell, then markdown cells will be ignored.)\n",
+ "2. A specially formatted markdown cell. The first line should be start with a single `#` (creating an H1 heading), and becomes the title. Then, optionally, a line beginning with `>` (creating a quote block), which becomes the description. Finally for `.ipynb` files (**NOT `.qmd` files**), zero or more lines beginning with `- ` (creating a list), each of which contains YAML. (If you already have \"title\" defined in frontmatter in a raw cell, then markdown cells will be ignored.)\n",
"\n",
"For instance, our test notebook contains the following markdown cell:\n",
"\n",
From e025097165631c95b4c93bbdcd4eb418c7080f8b Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Fri, 30 May 2025 21:15:28 -0400
Subject: [PATCH 16/31] Create script to convert .ipynb to .qmd
---
convert_nbs.py | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 98 insertions(+)
create mode 100644 convert_nbs.py
diff --git a/convert_nbs.py b/convert_nbs.py
new file mode 100644
index 000000000..bf26a031c
--- /dev/null
+++ b/convert_nbs.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+
+import os
+import shutil
+import re
+from pathlib import Path
+from execnb.nbio import read_nb, dict2nb
+from nbdev.process import read_qmd, write_qmd, read_nb_or_qmd
+from fastcore.script import call_parse
+
+
+@call_parse
+def convert_notebooks(
+ source_folder: str, # Source folder containing .ipynb files
+ dest_folder: str # Destination folder for .qmd files and copied items
+):
+ """
+ Converts .ipynb files from source_folder to .qmd files in dest_folder.
+ Other files are copied directly. Replicates directory structure.
+ """
+ source_dir = Path(source_folder)
+ dest_dir = Path(dest_folder)
+
+ if not source_dir.is_dir():
+ print(f"Error: Source directory '{source_dir.resolve()}' does not exist or is not a directory.")
+ return
+
+ print(f"Source directory: {source_dir.resolve()}")
+ print(f"Destination directory: {dest_dir.resolve()}")
+
+ dest_dir.mkdir(parents=True, exist_ok=True)
+ print(f"Ensured destination directory exists: {dest_dir}")
+
+ total_files_processed = 0
+ notebooks_converted = 0
+ files_copied = 0
+ errors_encountered = 0
+
+ for root, _, files in os.walk(source_dir):
+ current_source_dir = Path(root)
+ # Calculate path relative to the initial source_dir
+ # to replicate the structure in dest_dir
+ relative_subdir_path = current_source_dir.relative_to(source_dir)
+ current_dest_dir = dest_dir / relative_subdir_path
+
+ # Ensure the subdirectory structure exists in the destination
+ # (os.walk guarantees `root` exists, so mkdir for current_dest_dir is usually fine)
+ current_dest_dir.mkdir(parents=True, exist_ok=True)
+
+ for filename in files:
+ total_files_processed += 1
+ source_file_path = current_source_dir / filename
+
+ if source_file_path.name.startswith('.'): # Skip hidden files like .DS_Store
+ print(f"Skipping hidden file: {source_file_path}")
+ # Decrement count as we are not "processing" it in terms of conversion/copy
+ total_files_processed -=1
+ continue
+
+ if source_file_path.is_dir(): # Should not happen with os.walk's `files` list, but as a safeguard
+ print(f"Skipping directory listed as file: {source_file_path}")
+ total_files_processed -=1
+ continue
+
+ if source_file_path.suffix == ".ipynb":
+ # Prepare destination path for .qmd file
+ dest_qmd_filename = source_file_path.stem + ".qmd"
+ dest_file_path = current_dest_dir / dest_qmd_filename
+
+ print(f"Processing for conversion: {source_file_path} -> {dest_file_path}")
+ try:
+ # For .ipynb, read_nb_or_qmd will use read_nb from execnb
+ notebook_object = read_nb_or_qmd(source_file_path)
+ write_qmd(notebook_object, dest_file_path)
+ print(f" Successfully converted '{source_file_path.name}' to '{dest_file_path.name}'")
+ notebooks_converted +=1
+ except Exception as e:
+ print(f" Error converting {source_file_path}: {e}")
+ errors_encountered +=1
+ else:
+ # For any other file type, copy it directly
+ dest_file_path = current_dest_dir / filename
+ print(f"Copying: {source_file_path} -> {dest_file_path}")
+ try:
+ shutil.copy2(source_file_path, dest_file_path) # copy2 preserves metadata
+ print(f" Successfully copied '{source_file_path.name}'")
+ files_copied += 1
+ except Exception as e:
+ print(f" Error copying {source_file_path}: {e}")
+ errors_encountered +=1
+
+ print(f"\n--- Conversion Summary ---")
+ print(f"Total items scanned in source: {total_files_processed}")
+ print(f"Notebooks converted to .qmd: {notebooks_converted}")
+ print(f"Other files copied: {files_copied}")
+ if errors_encountered > 0:
+ print(f"Errors encountered: {errors_encountered}")
+ print(f"Output located in: {dest_dir.resolve()}")
\ No newline at end of file
From 19788db5b15ef1f66992555b6b99e482c800885e Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Fri, 30 May 2025 22:25:25 -0400
Subject: [PATCH 17/31] Add script to convert ipynb to qmd
---
nbdev/_modidx.py | 1 +
nbdev/qmd.py | 103 +++++++++++++++++++++++++++++++++++++-
nbs/api/15_qmd.ipynb | 116 ++++++++++++++++++++++++++++++++++++++++++-
settings.ini | 1 +
4 files changed, 218 insertions(+), 3 deletions(-)
diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py
index 5f282a13f..75423f352 100644
--- a/nbdev/_modidx.py
+++ b/nbdev/_modidx.py
@@ -246,6 +246,7 @@
'nbdev.qmd.btn': ('api/qmd.html#btn', 'nbdev/qmd.py'),
'nbdev.qmd.div': ('api/qmd.html#div', 'nbdev/qmd.py'),
'nbdev.qmd.img': ('api/qmd.html#img', 'nbdev/qmd.py'),
+ 'nbdev.qmd.ipynb_to_qmd': ('api/qmd.html#ipynb_to_qmd', 'nbdev/qmd.py'),
'nbdev.qmd.meta': ('api/qmd.html#meta', 'nbdev/qmd.py'),
'nbdev.qmd.read_nb_or_qmd': ('api/qmd.html#read_nb_or_qmd', 'nbdev/qmd.py'),
'nbdev.qmd.read_qmd': ('api/qmd.html#read_qmd', 'nbdev/qmd.py'),
diff --git a/nbdev/qmd.py b/nbdev/qmd.py
index d3b69227b..104611a2e 100644
--- a/nbdev/qmd.py
+++ b/nbdev/qmd.py
@@ -9,13 +9,15 @@
from pathlib import Path
import re
-import sys,os,inspect
+import sys,os,inspect,shutil
from fastcore.utils import *
+from fastcore.script import *
from fastcore.meta import delegates
# %% auto 0
-__all__ = ['read_qmd', 'read_nb_or_qmd', 'write_qmd', 'write_nb_or_qmd', 'meta', 'div', 'img', 'btn', 'tbl_row', 'tbl_sep']
+__all__ = ['read_qmd', 'read_nb_or_qmd', 'write_qmd', 'write_nb_or_qmd', 'meta', 'div', 'img', 'btn', 'tbl_row', 'tbl_sep',
+ 'ipynb_to_qmd']
# %% ../nbs/api/15_qmd.ipynb
def _qmd_to_raw_cell(source_str, cell_type_str, qmd_metadata=None):
@@ -171,3 +173,100 @@ def _install_nbdev():
conda install -c fastai nbdev
```
''', ['panel-tabset'])
+
+# %% ../nbs/api/15_qmd.ipynb
+@call_parse
+def ipynb_to_qmd(
+ source_folder: str, # Source folder containing .ipynb files
+ dest_folder: str # Destination folder for .qmd files and copied items
+):
+ """
+ Converts .ipynb files from source_folder to .qmd files in dest_folder.
+ Other files are copied directly. Replicates directory structure.
+
+ Warning, you will need to manually check the generated .qmd files for:
+ 1. **Code blocks that contain 3 backticks**.
+ All code blocks are exported using 3 backticks. Any codeblocks with python strings containing 3 consecutive backticks will break.
+ Manually change the code fences to use 4+ backticks if you need to include 3 consecutive backticks in a code block.
+
+ 2. **Frontmatter encoded using lists of KV pairs**.
+ This is not supported in .qmd files. You will need to manually add the frontmatter to the .qmd files in the standard frontmatter format.
+ """
+ source_dir = Path(source_folder)
+ dest_dir = Path(dest_folder)
+
+ if not source_dir.is_dir():
+ print(f"Error: Source directory '{source_dir.resolve()}' does not exist or is not a directory.")
+ return
+
+ print(f"Source directory: {source_dir.resolve()}")
+ print(f"Destination directory: {dest_dir.resolve()}")
+
+ dest_dir.mkdir(parents=True, exist_ok=True)
+ print(f"Ensured destination directory exists: {dest_dir}")
+
+ total_files_processed = 0
+ notebooks_converted = 0
+ files_copied = 0
+ errors_encountered = 0
+
+ for root, _, files in os.walk(source_dir):
+ current_source_dir = Path(root)
+ # Calculate path relative to the initial source_dir
+ # to replicate the structure in dest_dir
+ relative_subdir_path = current_source_dir.relative_to(source_dir)
+ current_dest_dir = dest_dir / relative_subdir_path
+
+ # Ensure the subdirectory structure exists in the destination
+ # (os.walk guarantees `root` exists, so mkdir for current_dest_dir is usually fine)
+ current_dest_dir.mkdir(parents=True, exist_ok=True)
+
+ for filename in files:
+ total_files_processed += 1
+ source_file_path = current_source_dir / filename
+
+ if source_file_path.name.startswith('.'): # Skip hidden files like .DS_Store
+ print(f"Skipping hidden file: {source_file_path}")
+ # Decrement count as we are not "processing" it in terms of conversion/copy
+ total_files_processed -=1
+ continue
+
+ if source_file_path.is_dir(): # Should not happen with os.walk's `files` list, but as a safeguard
+ print(f"Skipping directory listed as file: {source_file_path}")
+ total_files_processed -=1
+ continue
+
+ if source_file_path.suffix == ".ipynb":
+ # Prepare destination path for .qmd file
+ dest_qmd_filename = source_file_path.stem + ".qmd"
+ dest_file_path = current_dest_dir / dest_qmd_filename
+
+ print(f"Processing for conversion: {source_file_path} -> {dest_file_path}")
+ try:
+ # For .ipynb, read_nb_or_qmd will use read_nb from execnb
+ notebook_object = read_nb_or_qmd(source_file_path)
+ write_qmd(notebook_object, dest_file_path)
+ print(f" Successfully converted '{source_file_path.name}' to '{dest_file_path.name}'")
+ notebooks_converted +=1
+ except Exception as e:
+ print(f" Error converting {source_file_path}: {e}")
+ errors_encountered +=1
+ else:
+ # For any other file type, copy it directly
+ dest_file_path = current_dest_dir / filename
+ print(f"Copying: {source_file_path} -> {dest_file_path}")
+ try:
+ shutil.copy2(source_file_path, dest_file_path) # copy2 preserves metadata
+ print(f" Successfully copied '{source_file_path.name}'")
+ files_copied += 1
+ except Exception as e:
+ print(f" Error copying {source_file_path}: {e}")
+ errors_encountered +=1
+
+ print(f"\n--- Conversion Summary ---")
+ print(f"Total items scanned in source: {total_files_processed}")
+ print(f"Notebooks converted to .qmd: {notebooks_converted}")
+ print(f"Other files copied: {files_copied}")
+ if errors_encountered > 0:
+ print(f"Errors encountered: {errors_encountered}")
+ print(f"Output located in: {dest_dir.resolve()}")
diff --git a/nbs/api/15_qmd.ipynb b/nbs/api/15_qmd.ipynb
index a2ee56bfd..865a98b86 100644
--- a/nbs/api/15_qmd.ipynb
+++ b/nbs/api/15_qmd.ipynb
@@ -36,9 +36,10 @@
"from pathlib import Path\n",
"import re\n",
"\n",
- "import sys,os,inspect\n",
+ "import sys,os,inspect,shutil\n",
"\n",
"from fastcore.utils import *\n",
+ "from fastcore.script import *\n",
"from fastcore.meta import delegates"
]
},
@@ -355,6 +356,119 @@
"''', ['panel-tabset'])"
]
},
+ {
+ "cell_type": "markdown",
+ "id": "be8ac9d1",
+ "metadata": {},
+ "source": [
+ "Finally, we write a small helper function to convert a folder of .ipynb files to .qmd files."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7665a56d",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "#|export\n",
+ "@call_parse\n",
+ "def ipynb_to_qmd(\n",
+ " source_folder: str, # Source folder containing .ipynb files\n",
+ " dest_folder: str # Destination folder for .qmd files and copied items\n",
+ "):\n",
+ " \"\"\"\n",
+ " Converts .ipynb files from source_folder to .qmd files in dest_folder.\n",
+ " Other files are copied directly. Replicates directory structure.\n",
+ " \n",
+ " Warning, you will need to manually check the generated .qmd files for:\n",
+ " 1. **Code blocks that contain 3 backticks**. \n",
+ " All code blocks are exported using 3 backticks. Any codeblocks with python strings containing 3 consecutive backticks will break. \n",
+ " Manually change the code fences to use 4+ backticks if you need to include 3 consecutive backticks in a code block.\n",
+ " \n",
+ " 2. **Frontmatter encoded using lists of KV pairs**. \n",
+ " This is not supported in .qmd files. You will need to manually add the frontmatter to the .qmd files in the standard frontmatter format.\n",
+ " \"\"\"\n",
+ " source_dir = Path(source_folder)\n",
+ " dest_dir = Path(dest_folder)\n",
+ "\n",
+ " if not source_dir.is_dir():\n",
+ " print(f\"Error: Source directory '{source_dir.resolve()}' does not exist or is not a directory.\")\n",
+ " return\n",
+ "\n",
+ " print(f\"Source directory: {source_dir.resolve()}\")\n",
+ " print(f\"Destination directory: {dest_dir.resolve()}\")\n",
+ " \n",
+ " dest_dir.mkdir(parents=True, exist_ok=True)\n",
+ " print(f\"Ensured destination directory exists: {dest_dir}\")\n",
+ "\n",
+ " total_files_processed = 0\n",
+ " notebooks_converted = 0\n",
+ " files_copied = 0\n",
+ " errors_encountered = 0\n",
+ "\n",
+ " for root, _, files in os.walk(source_dir):\n",
+ " current_source_dir = Path(root)\n",
+ " # Calculate path relative to the initial source_dir\n",
+ " # to replicate the structure in dest_dir\n",
+ " relative_subdir_path = current_source_dir.relative_to(source_dir)\n",
+ " current_dest_dir = dest_dir / relative_subdir_path\n",
+ " \n",
+ " # Ensure the subdirectory structure exists in the destination\n",
+ " # (os.walk guarantees `root` exists, so mkdir for current_dest_dir is usually fine)\n",
+ " current_dest_dir.mkdir(parents=True, exist_ok=True)\n",
+ "\n",
+ " for filename in files:\n",
+ " total_files_processed += 1\n",
+ " source_file_path = current_source_dir / filename\n",
+ " \n",
+ " if source_file_path.name.startswith('.'): # Skip hidden files like .DS_Store\n",
+ " print(f\"Skipping hidden file: {source_file_path}\")\n",
+ " # Decrement count as we are not \"processing\" it in terms of conversion/copy\n",
+ " total_files_processed -=1 \n",
+ " continue\n",
+ "\n",
+ " if source_file_path.is_dir(): # Should not happen with os.walk's `files` list, but as a safeguard\n",
+ " print(f\"Skipping directory listed as file: {source_file_path}\")\n",
+ " total_files_processed -=1\n",
+ " continue\n",
+ "\n",
+ " if source_file_path.suffix == \".ipynb\":\n",
+ " # Prepare destination path for .qmd file\n",
+ " dest_qmd_filename = source_file_path.stem + \".qmd\"\n",
+ " dest_file_path = current_dest_dir / dest_qmd_filename\n",
+ " \n",
+ " print(f\"Processing for conversion: {source_file_path} -> {dest_file_path}\")\n",
+ " try:\n",
+ " # For .ipynb, read_nb_or_qmd will use read_nb from execnb\n",
+ " notebook_object = read_nb_or_qmd(source_file_path)\n",
+ " write_qmd(notebook_object, dest_file_path)\n",
+ " print(f\" Successfully converted '{source_file_path.name}' to '{dest_file_path.name}'\")\n",
+ " notebooks_converted +=1\n",
+ " except Exception as e:\n",
+ " print(f\" Error converting {source_file_path}: {e}\")\n",
+ " errors_encountered +=1\n",
+ " else:\n",
+ " # For any other file type, copy it directly\n",
+ " dest_file_path = current_dest_dir / filename\n",
+ " print(f\"Copying: {source_file_path} -> {dest_file_path}\")\n",
+ " try:\n",
+ " shutil.copy2(source_file_path, dest_file_path) # copy2 preserves metadata\n",
+ " print(f\" Successfully copied '{source_file_path.name}'\")\n",
+ " files_copied += 1\n",
+ " except Exception as e:\n",
+ " print(f\" Error copying {source_file_path}: {e}\")\n",
+ " errors_encountered +=1\n",
+ " \n",
+ " print(f\"\\n--- Conversion Summary ---\")\n",
+ " print(f\"Total items scanned in source: {total_files_processed}\")\n",
+ " print(f\"Notebooks converted to .qmd: {notebooks_converted}\")\n",
+ " print(f\"Other files copied: {files_copied}\")\n",
+ " if errors_encountered > 0:\n",
+ " print(f\"Errors encountered: {errors_encountered}\")\n",
+ " print(f\"Output located in: {dest_dir.resolve()}\") "
+ ]
+ },
{
"cell_type": "markdown",
"id": "aa35b010",
diff --git a/settings.ini b/settings.ini
index 535ff4b9c..1dc7955c3 100644
--- a/settings.ini
+++ b/settings.ini
@@ -50,6 +50,7 @@ console_scripts = nbdev_create_config=nbdev.config:nbdev_create_config
nbdev_bump_version=nbdev.release:nbdev_bump_version
nbdev_requirements=nbdev.release:write_requirements
nbdev_proc_nbs=nbdev.quarto:nbdev_proc_nbs
+ nbdev_ipynb_to_qmd=nbdev.qmd:ipynb_to_qmd
nbdev_help=nbdev.cli:chelp
nb_export=nbdev.cli:nb_export_cli
watch_export=nbdev.cli:watch_export
From 33ad5f14ca9ecd54186c8e0e8182ac6133bc1e64 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Sun, 1 Jun 2025 13:05:59 -0400
Subject: [PATCH 18/31] Allow markdown after custom frontmatter
---
.gitignore | 1 +
nbdev/frontmatter.py | 45 +++++++++++++++++++++++++-----------
nbs/api/09_frontmatter.ipynb | 36 ++++++++++++++++++-----------
3 files changed, 56 insertions(+), 26 deletions(-)
diff --git a/.gitignore b/.gitignore
index 347045c79..9c6eae719 100644
--- a/.gitignore
+++ b/.gitignore
@@ -150,3 +150,4 @@ checklink/cookies.txt
# .gitconfig is now autogenerated
.gitconfig
+nbdev_bak/
\ No newline at end of file
diff --git a/nbdev/frontmatter.py b/nbdev/frontmatter.py
index 440522776..b90985966 100644
--- a/nbdev/frontmatter.py
+++ b/nbdev/frontmatter.py
@@ -28,19 +28,24 @@ def _fm2dict(s:str, nb=True):
match = re_fm.search(s.strip())
return yaml.safe_load(match.group(1)) if match else {}
-def _md2dict(s:str, find_kv=True):
+def _md2dict(s:str):
"Convert H1 formatted markdown cell to frontmatter dict"
if '#' not in s: return {}
- m = re.search(r'^#\s+(\S.*?)\s*$', s, flags=re.MULTILINE)
- if not m: return {}
- res = {'title': m.group(1)}
- m = re.search(r'^>\s+(\S.*?)\s*$', s, flags=re.MULTILINE)
- if m: res['description'] = m.group(1)
- if find_kv:
- r = re.findall(r'^-\s+(\S.*:.*\S)\s*$', s, flags=re.MULTILINE)
- if r:
- try: res.update(yaml.safe_load('\n'.join(r)))
- except Exception as e: warn(f'Failed to create YAML dict for:\n{r}\n\n{e}\n')
+ # Captures frontmatter and any remaining content
+ pattern = r'^#\s+(\S.*?)\s*\n(?:\s*\n)*(?:>\s+(\S.*?)\s*\n(?:\s*\n)*((?:^\s*-\s+\S.*:.*\S\s*\n?)*))?\s*(.*?)$'
+ match = re.search(pattern, s.strip(), flags=re.MULTILINE | re.DOTALL)
+ if not match: return {}
+ res = {'title': match.group(1)}
+ if match.group(2): res['description'] = match.group(2)
+ if match.group(3):
+ kv_lines = re.findall(r'^-\s+(\S.*:.*\S)\s*$', match.group(3), flags=re.MULTILINE)
+ if kv_lines:
+ try: res.update(yaml.safe_load('\n'.join(kv_lines)))
+ except Exception as e: warn(f'Failed to create YAML dict for:\n{kv_lines}\n\n{e}\n')
+
+ # Add remaining content if present
+ remaining = match.group(4).strip()
+ if remaining: res['__remaining'] = remaining
return res
# %% ../nbs/api/09_frontmatter.ipynb
@@ -52,18 +57,32 @@ class FrontmatterProc(Processor):
def begin(self):
self.fm = getattr(self.nb, 'frontmatter_', {})
self.is_qmd = hasattr(self.nb, 'path_') and Path(self.nb.path_).suffix == '.qmd'
-
+
def _update(self, f, cell):
s = cell.get('source')
if not s: return
d = f(s)
if not d: return
+ remaining = d.pop('__remaining', None)
self.fm.update(d)
+ if remaining:
+ new_cell = mk_cell(remaining, 'markdown')
+ cell_idx = self.nb.cells.index(cell)
+ self.nb.cells.insert(cell_idx + 1, new_cell)
cell.source = None
+
+ # def _update(self, f, cell):
+ # s = cell.get('source')
+ # if not s: return
+ # d = f(s)
+ # if not d: return
+ # self.fm.update(d)
+ # if self.is_qmd: cell.source = None
+
def cell(self, cell):
if cell.cell_type=='raw': self._update(_fm2dict, cell)
- elif (cell.cell_type=='markdown' and 'title' not in self.fm): self._update(partial(_md2dict, find_kv=not self.is_qmd), cell)
+ elif (cell.cell_type=='markdown' and 'title' not in self.fm): self._update(_md2dict, cell)
def end(self):
self.nb.frontmatter_ = self.fm
diff --git a/nbs/api/09_frontmatter.ipynb b/nbs/api/09_frontmatter.ipynb
index e59ac5cb3..2331bbe79 100644
--- a/nbs/api/09_frontmatter.ipynb
+++ b/nbs/api/09_frontmatter.ipynb
@@ -81,19 +81,24 @@
" match = re_fm.search(s.strip())\n",
" return yaml.safe_load(match.group(1)) if match else {}\n",
"\n",
- "def _md2dict(s:str, find_kv=True):\n",
+ "def _md2dict(s:str):\n",
" \"Convert H1 formatted markdown cell to frontmatter dict\"\n",
" if '#' not in s: return {}\n",
- " m = re.search(r'^#\\s+(\\S.*?)\\s*$', s, flags=re.MULTILINE)\n",
- " if not m: return {}\n",
- " res = {'title': m.group(1)}\n",
- " m = re.search(r'^>\\s+(\\S.*?)\\s*$', s, flags=re.MULTILINE)\n",
- " if m: res['description'] = m.group(1)\n",
- " if find_kv:\n",
- " r = re.findall(r'^-\\s+(\\S.*:.*\\S)\\s*$', s, flags=re.MULTILINE)\n",
- " if r:\n",
- " try: res.update(yaml.safe_load('\\n'.join(r)))\n",
- " except Exception as e: warn(f'Failed to create YAML dict for:\\n{r}\\n\\n{e}\\n')\n",
+ " # Captures frontmatter and any remaining content\n",
+ " pattern = r'^#\\s+(\\S.*?)\\s*\\n(?:\\s*\\n)*(?:>\\s+(\\S.*?)\\s*\\n(?:\\s*\\n)*((?:^\\s*-\\s+\\S.*:.*\\S\\s*\\n?)*))?\\s*(.*?)$'\n",
+ " match = re.search(pattern, s.strip(), flags=re.MULTILINE | re.DOTALL)\n",
+ " if not match: return {}\n",
+ " res = {'title': match.group(1)}\n",
+ " if match.group(2): res['description'] = match.group(2)\n",
+ " if match.group(3):\n",
+ " kv_lines = re.findall(r'^-\\s+(\\S.*:.*\\S)\\s*$', match.group(3), flags=re.MULTILINE)\n",
+ " if kv_lines:\n",
+ " try: res.update(yaml.safe_load('\\n'.join(kv_lines)))\n",
+ " except Exception as e: warn(f'Failed to create YAML dict for:\\n{kv_lines}\\n\\n{e}\\n')\n",
+ " \n",
+ " # Add remaining content if present\n",
+ " remaining = match.group(4).strip()\n",
+ " if remaining: res['__remaining'] = remaining\n",
" return res"
]
},
@@ -113,18 +118,23 @@
" def begin(self): \n",
" self.fm = getattr(self.nb, 'frontmatter_', {})\n",
" self.is_qmd = hasattr(self.nb, 'path_') and Path(self.nb.path_).suffix == '.qmd'\n",
- "\n",
+ " \n",
" def _update(self, f, cell):\n",
" s = cell.get('source')\n",
" if not s: return\n",
" d = f(s)\n",
" if not d: return\n",
+ " remaining = d.pop('__remaining', None)\n",
" self.fm.update(d)\n",
+ " if remaining:\n",
+ " new_cell = mk_cell(remaining, 'markdown')\n",
+ " cell_idx = self.nb.cells.index(cell)\n",
+ " self.nb.cells.insert(cell_idx + 1, new_cell)\n",
" cell.source = None\n",
"\n",
" def cell(self, cell):\n",
" if cell.cell_type=='raw': self._update(_fm2dict, cell)\n",
- " elif (cell.cell_type=='markdown' and 'title' not in self.fm): self._update(partial(_md2dict, find_kv=not self.is_qmd), cell)\n",
+ " elif (cell.cell_type=='markdown' and 'title' not in self.fm): self._update(_md2dict, cell)\n",
"\n",
" def end(self):\n",
" self.nb.frontmatter_ = self.fm\n",
From b124c764749dfe43a06f00a3dcd6ad19a626f84a Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Sun, 1 Jun 2025 13:07:40 -0400
Subject: [PATCH 19/31] Pre-compile fm regex
---
nbdev/frontmatter.py | 15 ++++-----------
nbs/api/09_frontmatter.ipynb | 6 ++++--
2 files changed, 8 insertions(+), 13 deletions(-)
diff --git a/nbdev/frontmatter.py b/nbdev/frontmatter.py
index b90985966..8220cdb89 100644
--- a/nbdev/frontmatter.py
+++ b/nbdev/frontmatter.py
@@ -22,6 +22,9 @@
_re_fm_nb = re.compile(_RE_FM_BASE+'$', flags=re.DOTALL)
_re_fm_md = re.compile(_RE_FM_BASE, flags=re.DOTALL)
+_RE_FM_AND_MD = r'^#\s+(\S.*?)\s*\n(?:\s*\n)*(?:>\s+(\S.*?)\s*\n(?:\s*\n)*((?:^\s*-\s+\S.*:.*\S\s*\n?)*))?\s*(.*?)$'
+_re_fm_and_md = re.compile(_RE_FM_AND_MD, flags=re.MULTILINE | re.DOTALL)
+
def _fm2dict(s:str, nb=True):
"Load YAML frontmatter into a `dict`"
re_fm = _re_fm_nb if nb else _re_fm_md
@@ -32,8 +35,7 @@ def _md2dict(s:str):
"Convert H1 formatted markdown cell to frontmatter dict"
if '#' not in s: return {}
# Captures frontmatter and any remaining content
- pattern = r'^#\s+(\S.*?)\s*\n(?:\s*\n)*(?:>\s+(\S.*?)\s*\n(?:\s*\n)*((?:^\s*-\s+\S.*:.*\S\s*\n?)*))?\s*(.*?)$'
- match = re.search(pattern, s.strip(), flags=re.MULTILINE | re.DOTALL)
+ match = _re_fm_and_md.search(s.strip())
if not match: return {}
res = {'title': match.group(1)}
if match.group(2): res['description'] = match.group(2)
@@ -71,15 +73,6 @@ def _update(self, f, cell):
self.nb.cells.insert(cell_idx + 1, new_cell)
cell.source = None
-
- # def _update(self, f, cell):
- # s = cell.get('source')
- # if not s: return
- # d = f(s)
- # if not d: return
- # self.fm.update(d)
- # if self.is_qmd: cell.source = None
-
def cell(self, cell):
if cell.cell_type=='raw': self._update(_fm2dict, cell)
elif (cell.cell_type=='markdown' and 'title' not in self.fm): self._update(_md2dict, cell)
diff --git a/nbs/api/09_frontmatter.ipynb b/nbs/api/09_frontmatter.ipynb
index 2331bbe79..9be338500 100644
--- a/nbs/api/09_frontmatter.ipynb
+++ b/nbs/api/09_frontmatter.ipynb
@@ -75,6 +75,9 @@
"_re_fm_nb = re.compile(_RE_FM_BASE+'$', flags=re.DOTALL)\n",
"_re_fm_md = re.compile(_RE_FM_BASE, flags=re.DOTALL)\n",
"\n",
+ "_RE_FM_AND_MD = r'^#\\s+(\\S.*?)\\s*\\n(?:\\s*\\n)*(?:>\\s+(\\S.*?)\\s*\\n(?:\\s*\\n)*((?:^\\s*-\\s+\\S.*:.*\\S\\s*\\n?)*))?\\s*(.*?)$'\n",
+ "_re_fm_and_md = re.compile(_RE_FM_AND_MD, flags=re.MULTILINE | re.DOTALL)\n",
+ "\n",
"def _fm2dict(s:str, nb=True):\n",
" \"Load YAML frontmatter into a `dict`\"\n",
" re_fm = _re_fm_nb if nb else _re_fm_md\n",
@@ -85,8 +88,7 @@
" \"Convert H1 formatted markdown cell to frontmatter dict\"\n",
" if '#' not in s: return {}\n",
" # Captures frontmatter and any remaining content\n",
- " pattern = r'^#\\s+(\\S.*?)\\s*\\n(?:\\s*\\n)*(?:>\\s+(\\S.*?)\\s*\\n(?:\\s*\\n)*((?:^\\s*-\\s+\\S.*:.*\\S\\s*\\n?)*))?\\s*(.*?)$'\n",
- " match = re.search(pattern, s.strip(), flags=re.MULTILINE | re.DOTALL)\n",
+ " match = _re_fm_and_md.search(s.strip())\n",
" if not match: return {}\n",
" res = {'title': match.group(1)}\n",
" if match.group(2): res['description'] = match.group(2)\n",
From 33a10cbc3eede8258f168faf0336869625c539e6 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Sun, 1 Jun 2025 13:45:04 -0400
Subject: [PATCH 20/31] Improve converting from .ipynb to .qmd (account for
backticks in python cells)
---
nbdev/_modidx.py | 3 ++-
nbdev/qmd.py | 17 ++++++++++++++++-
nbs/api/15_qmd.ipynb | 17 ++++++++++++++++-
3 files changed, 34 insertions(+), 3 deletions(-)
diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py
index 75423f352..ee0530554 100644
--- a/nbdev/_modidx.py
+++ b/nbdev/_modidx.py
@@ -240,7 +240,8 @@
'nbdev.processors.strip_ansi': ('api/processors.html#strip_ansi', 'nbdev/processors.py'),
'nbdev.processors.strip_hidden_metadata': ( 'api/processors.html#strip_hidden_metadata',
'nbdev/processors.py')},
- 'nbdev.qmd': { 'nbdev.qmd._install_nbdev': ('api/qmd.html#_install_nbdev', 'nbdev/qmd.py'),
+ 'nbdev.qmd': { 'nbdev.qmd._get_fence_ticks': ('api/qmd.html#_get_fence_ticks', 'nbdev/qmd.py'),
+ 'nbdev.qmd._install_nbdev': ('api/qmd.html#_install_nbdev', 'nbdev/qmd.py'),
'nbdev.qmd._nb_to_qmd_str': ('api/qmd.html#_nb_to_qmd_str', 'nbdev/qmd.py'),
'nbdev.qmd._qmd_to_raw_cell': ('api/qmd.html#_qmd_to_raw_cell', 'nbdev/qmd.py'),
'nbdev.qmd.btn': ('api/qmd.html#btn', 'nbdev/qmd.py'),
diff --git a/nbdev/qmd.py b/nbdev/qmd.py
index 104611a2e..cada59135 100644
--- a/nbdev/qmd.py
+++ b/nbdev/qmd.py
@@ -74,15 +74,30 @@ def read_nb_or_qmd(path):
# %% ../nbs/api/15_qmd.ipynb
+def _get_fence_ticks(source):
+ """Determine the number of backticks needed for fencing that won't conflict with source"""
+ if '`' not in source: return '```' # Default to 3 if no backticks in source
+
+ # Find all sequences of consecutive backticks
+ import re
+ backtick_sequences = re.findall(r'^`{3,}\s*$', source, flags=re.MULTILINE)
+ if not backtick_sequences: return '```'
+ used_lengths = set(len(s) for s in backtick_sequences)
+ # Find first integer >= 3 that's not in used_lengths
+ num_ticks = 3
+ while num_ticks in used_lengths: num_ticks += 1
+ return '`' * num_ticks
+
def _nb_to_qmd_str(nb):
"""Convert a notebook to a string in .qmd format"""
def cell_to_qmd(cell):
source = cell.source.rstrip('\n')
if cell.cell_type in ['markdown', 'raw']: return source
elif cell.cell_type == 'code':
+ fence_ticks = _get_fence_ticks(source)
qmd_metadata = getattr(cell, 'qmd_metadata', None)
if qmd_metadata: return f'```{{python{qmd_metadata}}}\n{source}\n```'
- else: return f'```{{python}}\n{source}\n```'
+ else: return f'{fence_ticks}{{python}}\n{source}\n{fence_ticks}'
return ''
return '\n\n'.join(filter(None, [cell_to_qmd(cell) for cell in nb.cells]))
diff --git a/nbs/api/15_qmd.ipynb b/nbs/api/15_qmd.ipynb
index 865a98b86..7cc6b4a02 100644
--- a/nbs/api/15_qmd.ipynb
+++ b/nbs/api/15_qmd.ipynb
@@ -200,15 +200,30 @@
"outputs": [],
"source": [
"#|export\n",
+ "def _get_fence_ticks(source):\n",
+ " \"\"\"Determine the number of backticks needed for fencing that won't conflict with source\"\"\"\n",
+ " if '`' not in source: return '```' # Default to 3 if no backticks in source\n",
+ " \n",
+ " # Find all sequences of consecutive backticks\n",
+ " import re\n",
+ " backtick_sequences = re.findall(r'^`{3,}\\s*$', source, flags=re.MULTILINE)\n",
+ " if not backtick_sequences: return '```'\n",
+ " used_lengths = set(len(s) for s in backtick_sequences)\n",
+ " # Find first integer >= 3 that's not in used_lengths\n",
+ " num_ticks = 3\n",
+ " while num_ticks in used_lengths: num_ticks += 1\n",
+ " return '`' * num_ticks\n",
+ "\n",
"def _nb_to_qmd_str(nb):\n",
" \"\"\"Convert a notebook to a string in .qmd format\"\"\"\n",
" def cell_to_qmd(cell):\n",
" source = cell.source.rstrip('\\n')\n",
" if cell.cell_type in ['markdown', 'raw']: return source\n",
" elif cell.cell_type == 'code':\n",
+ " fence_ticks = _get_fence_ticks(source)\n",
" qmd_metadata = getattr(cell, 'qmd_metadata', None)\n",
" if qmd_metadata: return f'```{{python{qmd_metadata}}}\\n{source}\\n```'\n",
- " else: return f'```{{python}}\\n{source}\\n```'\n",
+ " else: return f'{fence_ticks}{{python}}\\n{source}\\n{fence_ticks}'\n",
" return ''\n",
" return '\\n\\n'.join(filter(None, [cell_to_qmd(cell) for cell in nb.cells]))\n",
"\n",
From bdfb1dbc95c3be99c6fe41561f26469658036259 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Sun, 1 Jun 2025 14:53:46 -0400
Subject: [PATCH 21/31] .qmd files now read custom KV yaml
---
nbdev/frontmatter.py | 4 ++--
nbdev/maker.py | 1 +
nbs/api/02_maker.ipynb | 3 ++-
nbs/api/09_frontmatter.ipynb | 13 ++++++++++---
4 files changed, 15 insertions(+), 6 deletions(-)
diff --git a/nbdev/frontmatter.py b/nbdev/frontmatter.py
index 8220cdb89..974937b9d 100644
--- a/nbdev/frontmatter.py
+++ b/nbdev/frontmatter.py
@@ -22,7 +22,7 @@
_re_fm_nb = re.compile(_RE_FM_BASE+'$', flags=re.DOTALL)
_re_fm_md = re.compile(_RE_FM_BASE, flags=re.DOTALL)
-_RE_FM_AND_MD = r'^#\s+(\S.*?)\s*\n(?:\s*\n)*(?:>\s+(\S.*?)\s*\n(?:\s*\n)*((?:^\s*-\s+\S.*:.*\S\s*\n?)*))?\s*(.*?)$'
+_RE_FM_AND_MD = r'^#\s+(\S.*?)\s*\n(?:\s*\n)*(?:>\s+(\S.*?)\s*\n(?:\s*\n)*((?:^\s*-\s+[a-zA-Z_][a-zA-Z0-9_]*\s*:\s+.*\s*\n?)*))?\s*(.*?)$'
_re_fm_and_md = re.compile(_RE_FM_AND_MD, flags=re.MULTILINE | re.DOTALL)
def _fm2dict(s:str, nb=True):
@@ -40,7 +40,7 @@ def _md2dict(s:str):
res = {'title': match.group(1)}
if match.group(2): res['description'] = match.group(2)
if match.group(3):
- kv_lines = re.findall(r'^-\s+(\S.*:.*\S)\s*$', match.group(3), flags=re.MULTILINE)
+ kv_lines = re.findall(r'^-\s+([a-zA-Z_][a-zA-Z0-9_]*\s*:\s+.*)\s*$', match.group(3), flags=re.MULTILINE)
if kv_lines:
try: res.update(yaml.safe_load('\n'.join(kv_lines)))
except Exception as e: warn(f'Failed to create YAML dict for:\n{kv_lines}\n\n{e}\n')
diff --git a/nbdev/maker.py b/nbdev/maker.py
index 277895805..2e17ac25c 100644
--- a/nbdev/maker.py
+++ b/nbdev/maker.py
@@ -11,6 +11,7 @@
# %% ../nbs/api/02_maker.ipynb
from .config import *
from .imports import *
+from .qmd import *
from fastcore.script import *
from fastcore.basics import *
diff --git a/nbs/api/02_maker.ipynb b/nbs/api/02_maker.ipynb
index 259812444..5271c2e66 100644
--- a/nbs/api/02_maker.ipynb
+++ b/nbs/api/02_maker.ipynb
@@ -38,6 +38,7 @@
"#|export\n",
"from nbdev.config import *\n",
"from nbdev.imports import *\n",
+ "from nbdev.qmd import *\n",
"\n",
"from fastcore.script import *\n",
"from fastcore.basics import *\n",
@@ -485,7 +486,7 @@
"outputs": [],
"source": [
"#|hide\n",
- "nb = read_nb('02_maker.ipynb')\n",
+ "nb = read_nb_or_qmd('02_maker.ipynb')\n",
"test_eq(_retr_mdoc(nb.cells), '\"\"\"Create one or more modules from selected notebook cells\"\"\"\\n\\n')"
]
},
diff --git a/nbs/api/09_frontmatter.ipynb b/nbs/api/09_frontmatter.ipynb
index 9be338500..00189382a 100644
--- a/nbs/api/09_frontmatter.ipynb
+++ b/nbs/api/09_frontmatter.ipynb
@@ -75,7 +75,7 @@
"_re_fm_nb = re.compile(_RE_FM_BASE+'$', flags=re.DOTALL)\n",
"_re_fm_md = re.compile(_RE_FM_BASE, flags=re.DOTALL)\n",
"\n",
- "_RE_FM_AND_MD = r'^#\\s+(\\S.*?)\\s*\\n(?:\\s*\\n)*(?:>\\s+(\\S.*?)\\s*\\n(?:\\s*\\n)*((?:^\\s*-\\s+\\S.*:.*\\S\\s*\\n?)*))?\\s*(.*?)$'\n",
+ "_RE_FM_AND_MD = r'^#\\s+(\\S.*?)\\s*\\n(?:\\s*\\n)*(?:>\\s+(\\S.*?)\\s*\\n(?:\\s*\\n)*((?:^\\s*-\\s+[a-zA-Z_][a-zA-Z0-9_]*\\s*:\\s+.*\\s*\\n?)*))?\\s*(.*?)$'\n",
"_re_fm_and_md = re.compile(_RE_FM_AND_MD, flags=re.MULTILINE | re.DOTALL)\n",
"\n",
"def _fm2dict(s:str, nb=True):\n",
@@ -93,7 +93,7 @@
" res = {'title': match.group(1)}\n",
" if match.group(2): res['description'] = match.group(2)\n",
" if match.group(3):\n",
- " kv_lines = re.findall(r'^-\\s+(\\S.*:.*\\S)\\s*$', match.group(3), flags=re.MULTILINE)\n",
+ " kv_lines = re.findall(r'^-\\s+([a-zA-Z_][a-zA-Z0-9_]*\\s*:\\s+.*)\\s*$', match.group(3), flags=re.MULTILINE)\n",
" if kv_lines:\n",
" try: res.update(yaml.safe_load('\\n'.join(kv_lines)))\n",
" except Exception as e: warn(f'Failed to create YAML dict for:\\n{kv_lines}\\n\\n{e}\\n')\n",
@@ -158,7 +158,7 @@
"YAML frontmatter can be added to notebooks in one of two ways:\n",
"\n",
"1. By adding a raw notebook cell with `---` as the first and last lines, and YAML between them, or\n",
- "2. A specially formatted markdown cell. The first line should be start with a single `#` (creating an H1 heading), and becomes the title. Then, optionally, a line beginning with `>` (creating a quote block), which becomes the description. Finally for `.ipynb` files (**NOT `.qmd` files**), zero or more lines beginning with `- ` (creating a list), each of which contains YAML. (If you already have \"title\" defined in frontmatter in a raw cell, then markdown cells will be ignored.)\n",
+ "2. A specially formatted markdown cell. The first line should be start with a single `#` (creating an H1 heading), and becomes the title. Then, optionally, a line beginning with `>` (creating a quote block), which becomes the description. Finally, zero or more lines beginning with `- ` (creating a list), each of which contains YAML key-value pairs. (If you already have \"title\" defined in frontmatter in a raw cell, then markdown cells will be ignored.)\n",
"\n",
"For instance, our test notebook contains the following markdown cell:\n",
"\n",
@@ -182,6 +182,13 @@
"When we process with `FrontmatterProc`, these will both be removed, and a single raw cell will be added to the top, containing the combined YAML frontmatter:"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We similarly need a `write_qmd` function to write a notebook to a .qmd file."
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
From 86354d92ef0b81413cc8ede086677a64898bcd27 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Sun, 1 Jun 2025 23:04:44 -0400
Subject: [PATCH 22/31] Custom frontmatter now parsed correctly for docs
---
README.md | 15 +++++
nbdev/_modidx.py | 3 +-
nbdev/frontmatter.py | 83 ++++++++++++++++++++-------
nbs/api/09_frontmatter.ipynb | 106 ++++++++++++++++++++++++++++-------
nbs/api/16_migrate.ipynb | 17 +++++-
tests/docs_test.ipynb | 25 ++++++++-
6 files changed, 204 insertions(+), 45 deletions(-)
diff --git a/README.md b/README.md
index ba5728950..870eaf572 100644
--- a/README.md
+++ b/README.md
@@ -79,10 +79,12 @@ available commands:
!nbdev_help
```
+ nb_export Export a single nbdev notebook to a python script.
nbdev_bump_version Increment version in settings.ini by one
nbdev_changelog Create a CHANGELOG.md file from closed and labeled GitHub issues
nbdev_clean Clean all notebooks in `fname` to avoid merge conflicts
nbdev_conda Create a `meta.yaml` file ready to be built into a package, and optionally build and upload it
+ nbdev_contributing Create CONTRIBUTING.md from contributing_nb (defaults to 'contributing.ipynb' if present). Skips if the file doesn't exist.
nbdev_create_config Create a config file.
nbdev_docs Create Quarto docs and README.md
nbdev_export Export notebooks in `path` to Python modules
@@ -92,6 +94,18 @@ available commands:
nbdev_install Install Quarto and the current library
nbdev_install_hooks Install Jupyter and git hooks to automatically clean, trust, and fix merge conflicts in notebooks
nbdev_install_quarto Install latest Quarto on macOS or Linux, prints instructions for Windows
+ nbdev_ipynb_to_qmd
+ Converts .ipynb files from source_folder to .qmd files in dest_folder.
+ Other files are copied directly. Replicates directory structure.
+
+ Warning, you will need to manually check the generated .qmd files for:
+ 1. **Code blocks that contain 3 backticks**.
+ All code blocks are exported using 3 backticks. Any codeblocks with python strings containing 3 consecutive backticks will break.
+ Manually change the code fences to use 4+ backticks if you need to include 3 consecutive backticks in a code block.
+
+ 2. **Frontmatter encoded using lists of KV pairs**.
+ This is not supported in .qmd files. You will need to manually add the frontmatter to the .qmd files in the standard frontmatter format.
+
nbdev_merge Git merge driver for notebooks
nbdev_migrate Convert all markdown and notebook files in `path` from v1 to v2
nbdev_new Create an nbdev project.
@@ -109,6 +123,7 @@ available commands:
nbdev_trust Trust notebooks matching `fname`
nbdev_update Propagate change in modules matching `fname` to notebooks that created them
nbdev_update_license Allows you to update the license of your project.
+ watch_export Use `nb_export` on ipynb files in `nbs` directory on changes using nbdev config if available
## FAQ
diff --git a/nbdev/_modidx.py b/nbdev/_modidx.py
index ee0530554..bd051adad 100644
--- a/nbdev/_modidx.py
+++ b/nbdev/_modidx.py
@@ -100,7 +100,8 @@
'nbdev.frontmatter._dict2fm': ('api/frontmatter.html#_dict2fm', 'nbdev/frontmatter.py'),
'nbdev.frontmatter._fm2dict': ('api/frontmatter.html#_fm2dict', 'nbdev/frontmatter.py'),
'nbdev.frontmatter._insertfm': ('api/frontmatter.html#_insertfm', 'nbdev/frontmatter.py'),
- 'nbdev.frontmatter._md2dict': ('api/frontmatter.html#_md2dict', 'nbdev/frontmatter.py')},
+ 'nbdev.frontmatter._md2dict': ('api/frontmatter.html#_md2dict', 'nbdev/frontmatter.py'),
+ 'nbdev.frontmatter._parse_kv_block': ('api/frontmatter.html#_parse_kv_block', 'nbdev/frontmatter.py')},
'nbdev.imports': {},
'nbdev.maker': { 'nbdev.maker.ModuleMaker': ('api/maker.html#modulemaker', 'nbdev/maker.py'),
'nbdev.maker.ModuleMaker.__init__': ('api/maker.html#modulemaker.__init__', 'nbdev/maker.py'),
diff --git a/nbdev/frontmatter.py b/nbdev/frontmatter.py
index 974937b9d..97dfa5ffc 100644
--- a/nbdev/frontmatter.py
+++ b/nbdev/frontmatter.py
@@ -22,34 +22,77 @@
_re_fm_nb = re.compile(_RE_FM_BASE+'$', flags=re.DOTALL)
_re_fm_md = re.compile(_RE_FM_BASE, flags=re.DOTALL)
-_RE_FM_AND_MD = r'^#\s+(\S.*?)\s*\n(?:\s*\n)*(?:>\s+(\S.*?)\s*\n(?:\s*\n)*((?:^\s*-\s+[a-zA-Z_][a-zA-Z0-9_]*\s*:\s+.*\s*\n?)*))?\s*(.*?)$'
-_re_fm_and_md = re.compile(_RE_FM_AND_MD, flags=re.MULTILINE | re.DOTALL)
-def _fm2dict(s:str, nb=True):
- "Load YAML frontmatter into a `dict`"
- re_fm = _re_fm_nb if nb else _re_fm_md
- match = re_fm.search(s.strip())
- return yaml.safe_load(match.group(1)) if match else {}
+# Regex 1: Capture title and description only
+_re_fm_title_desc = re.compile(r'^#\s+(\S.*?)(?:\n|$)(?:\s*\n)*(?:>\s+(\S.*?)(?:\n|$)(?:\s*\n)*)?', flags=re.MULTILINE)
+
+# Regex 2: Capture contiguous KV block (lines starting with - or blank lines)
+_re_fm_kv = re.compile(r'^((?:\s*-\s+[a-zA-Z_][a-zA-Z0-9_]*\s*:\s+.*(?:\n|$)|\s*\n)*)', flags=re.MULTILINE)
+
+def _parse_kv_block(block_text):
+ """Parse a block of key-value pairs from lines starting with '-'"""
+ if not block_text.strip():
+ return {}
+
+ # Extract only the lines that start with '-' (ignore blank lines)
+ kv_lines = []
+ for line in block_text.split('\n'):
+ line = line.strip()
+ if line.startswith('-'):
+ kv_content = line[1:].strip() # Remove the '-' and strip whitespace
+ if kv_content:
+ kv_lines.append(kv_content)
+
+ if not kv_lines:
+ return {}
+
+ try:
+ return yaml.safe_load('\n'.join(kv_lines))
+ except Exception as e:
+ warn(f'Failed to create YAML dict for:\n{kv_lines}\n\n{e}\n')
+ return {}
def _md2dict(s:str):
"Convert H1 formatted markdown cell to frontmatter dict"
if '#' not in s: return {}
- # Captures frontmatter and any remaining content
- match = _re_fm_and_md.search(s.strip())
- if not match: return {}
- res = {'title': match.group(1)}
- if match.group(2): res['description'] = match.group(2)
- if match.group(3):
- kv_lines = re.findall(r'^-\s+([a-zA-Z_][a-zA-Z0-9_]*\s*:\s+.*)\s*$', match.group(3), flags=re.MULTILINE)
- if kv_lines:
- try: res.update(yaml.safe_load('\n'.join(kv_lines)))
- except Exception as e: warn(f'Failed to create YAML dict for:\n{kv_lines}\n\n{e}\n')
- # Add remaining content if present
- remaining = match.group(4).strip()
- if remaining: res['__remaining'] = remaining
+ res = {}
+ remaining_start = 0
+
+ # Step 1: Extract title and description
+ title_desc_match = _re_fm_title_desc.match(s)
+ if not title_desc_match:
+ return {}
+
+ res['title'] = title_desc_match.group(1)
+ if title_desc_match.group(2):
+ res['description'] = title_desc_match.group(2)
+
+ remaining_start = title_desc_match.end()
+
+ # Step 2: Extract KV pairs starting from where title/desc ended
+ kv_text = s[remaining_start:]
+ kv_match = _re_fm_kv.match(kv_text)
+ if kv_match:
+ kv_dict = _parse_kv_block(kv_match.group(1))
+ res.update(kv_dict)
+ remaining_start += kv_match.end()
+
+ # Step 3: Everything else is remaining content
+ remaining = s[remaining_start:].strip()
+ if remaining:
+ # print("REMAINING ", remaining)
+ # print("Total string", s)
+ res['__remaining'] = remaining
+
return res
+def _fm2dict(s:str, nb=True):
+ "Load YAML frontmatter into a `dict`"
+ re_fm = _re_fm_nb if nb else _re_fm_md
+ match = re_fm.search(s.strip())
+ return yaml.safe_load(match.group(1)) if match else {}
+
# %% ../nbs/api/09_frontmatter.ipynb
def _dict2fm(d): return f'---\n{yaml.dump(d)}\n---\n\n'
def _insertfm(nb, fm): nb.cells.insert(0, mk_cell(_dict2fm(fm), 'raw'))
diff --git a/nbs/api/09_frontmatter.ipynb b/nbs/api/09_frontmatter.ipynb
index 00189382a..6acb2060c 100644
--- a/nbs/api/09_frontmatter.ipynb
+++ b/nbs/api/09_frontmatter.ipynb
@@ -75,33 +75,76 @@
"_re_fm_nb = re.compile(_RE_FM_BASE+'$', flags=re.DOTALL)\n",
"_re_fm_md = re.compile(_RE_FM_BASE, flags=re.DOTALL)\n",
"\n",
- "_RE_FM_AND_MD = r'^#\\s+(\\S.*?)\\s*\\n(?:\\s*\\n)*(?:>\\s+(\\S.*?)\\s*\\n(?:\\s*\\n)*((?:^\\s*-\\s+[a-zA-Z_][a-zA-Z0-9_]*\\s*:\\s+.*\\s*\\n?)*))?\\s*(.*?)$'\n",
- "_re_fm_and_md = re.compile(_RE_FM_AND_MD, flags=re.MULTILINE | re.DOTALL)\n",
"\n",
- "def _fm2dict(s:str, nb=True):\n",
- " \"Load YAML frontmatter into a `dict`\"\n",
- " re_fm = _re_fm_nb if nb else _re_fm_md\n",
- " match = re_fm.search(s.strip())\n",
- " return yaml.safe_load(match.group(1)) if match else {}\n",
+ "# Regex 1: Capture title and description only\n",
+ "_re_fm_title_desc = re.compile(r'^#\\s+(\\S.*?)(?:\\n|$)(?:\\s*\\n)*(?:>\\s+(\\S.*?)(?:\\n|$)(?:\\s*\\n)*)?', flags=re.MULTILINE)\n",
+ "\n",
+ "# Regex 2: Capture contiguous KV block (lines starting with - or blank lines)\n",
+ "_re_fm_kv = re.compile(r'^((?:\\s*-\\s+[a-zA-Z_][a-zA-Z0-9_]*\\s*:\\s+.*(?:\\n|$)|\\s*\\n)*)', flags=re.MULTILINE)\n",
+ "\n",
+ "def _parse_kv_block(block_text):\n",
+ " \"\"\"Parse a block of key-value pairs from lines starting with '-'\"\"\"\n",
+ " if not block_text.strip():\n",
+ " return {}\n",
+ " \n",
+ " # Extract only the lines that start with '-' (ignore blank lines)\n",
+ " kv_lines = []\n",
+ " for line in block_text.split('\\n'):\n",
+ " line = line.strip()\n",
+ " if line.startswith('-'):\n",
+ " kv_content = line[1:].strip() # Remove the '-' and strip whitespace\n",
+ " if kv_content:\n",
+ " kv_lines.append(kv_content)\n",
+ " \n",
+ " if not kv_lines:\n",
+ " return {}\n",
+ " \n",
+ " try:\n",
+ " return yaml.safe_load('\\n'.join(kv_lines))\n",
+ " except Exception as e:\n",
+ " warn(f'Failed to create YAML dict for:\\n{kv_lines}\\n\\n{e}\\n')\n",
+ " return {}\n",
"\n",
"def _md2dict(s:str):\n",
" \"Convert H1 formatted markdown cell to frontmatter dict\"\n",
" if '#' not in s: return {}\n",
- " # Captures frontmatter and any remaining content\n",
- " match = _re_fm_and_md.search(s.strip())\n",
- " if not match: return {}\n",
- " res = {'title': match.group(1)}\n",
- " if match.group(2): res['description'] = match.group(2)\n",
- " if match.group(3):\n",
- " kv_lines = re.findall(r'^-\\s+([a-zA-Z_][a-zA-Z0-9_]*\\s*:\\s+.*)\\s*$', match.group(3), flags=re.MULTILINE)\n",
- " if kv_lines:\n",
- " try: res.update(yaml.safe_load('\\n'.join(kv_lines)))\n",
- " except Exception as e: warn(f'Failed to create YAML dict for:\\n{kv_lines}\\n\\n{e}\\n')\n",
" \n",
- " # Add remaining content if present\n",
- " remaining = match.group(4).strip()\n",
- " if remaining: res['__remaining'] = remaining\n",
- " return res"
+ " res = {}\n",
+ " remaining_start = 0\n",
+ " \n",
+ " # Step 1: Extract title and description\n",
+ " title_desc_match = _re_fm_title_desc.match(s)\n",
+ " if not title_desc_match:\n",
+ " return {}\n",
+ " \n",
+ " res['title'] = title_desc_match.group(1)\n",
+ " if title_desc_match.group(2):\n",
+ " res['description'] = title_desc_match.group(2)\n",
+ " \n",
+ " remaining_start = title_desc_match.end()\n",
+ " \n",
+ " # Step 2: Extract KV pairs starting from where title/desc ended\n",
+ " kv_text = s[remaining_start:]\n",
+ " kv_match = _re_fm_kv.match(kv_text)\n",
+ " if kv_match:\n",
+ " kv_dict = _parse_kv_block(kv_match.group(1))\n",
+ " res.update(kv_dict)\n",
+ " remaining_start += kv_match.end()\n",
+ " \n",
+ " # Step 3: Everything else is remaining content\n",
+ " remaining = s[remaining_start:].strip()\n",
+ " if remaining:\n",
+ " # print(\"REMAINING \", remaining)\n",
+ " # print(\"Total string\", s)\n",
+ " res['__remaining'] = remaining\n",
+ " \n",
+ " return res\n",
+ "\n",
+ "def _fm2dict(s:str, nb=True):\n",
+ " \"Load YAML frontmatter into a `dict`\"\n",
+ " re_fm = _re_fm_nb if nb else _re_fm_md\n",
+ " match = re_fm.search(s.strip())\n",
+ " return yaml.safe_load(match.group(1)) if match else {}"
]
},
{
@@ -184,6 +227,7 @@
},
{
"cell_type": "markdown",
+ "id": "9a0d6f22",
"metadata": {},
"source": [
"We similarly need a `write_qmd` function to write a notebook to a .qmd file."
@@ -199,6 +243,26 @@
"name": "stdout",
"output_type": "stream",
"text": [
+ "REMAINING With some remaining text that I want to preserve!!\n",
+ "\n",
+ "E.g., see this list:\n",
+ "\n",
+ "- [With a link](https://example.com)\n",
+ "\n",
+ "And this table:\n",
+ "\n",
+ "| Column 1 | Column 2 |\n",
+ "|----------|----------|\n",
+ "\n",
+ "and this example code:\n",
+ "\n",
+ "```python\n",
+ "a = 4 + 6\n",
+ "```\n",
+ "\n",
+ "And this image:\n",
+ "\n",
+ "\n",
"---\n",
"categories:\n",
"- c1\n",
diff --git a/nbs/api/16_migrate.ipynb b/nbs/api/16_migrate.ipynb
index 8c318f267..6d3a41ba7 100644
--- a/nbs/api/16_migrate.ipynb
+++ b/nbs/api/16_migrate.ipynb
@@ -306,7 +306,6 @@
"aliases:\n",
"- /fastcore/\n",
"author: Hamel Husain\n",
- "badges: true\n",
"categories:\n",
"- fastcore\n",
"- fastai\n",
@@ -339,7 +338,21 @@
"execution_count": null,
"id": "2010dbfc-be80-428c-a9cf-dd8fc80ac972",
"metadata": {},
- "outputs": [],
+ "outputs": [
+ {
+ "ename": "AssertionError",
+ "evalue": "==:\n---\naliases:\n- /fastcore/\nauthor: Hamel Husain\ncategories:\n- fastcore\n- fastai\ndate: '2020-09-01'\ndescription: A unique python library that extends the python programming language\n and provides utilities that enhance productivity.\ndraft: 'true'\nimage: fastcore_imgs/td.png\noutput-file: 2020-09-01-fastcore.html\npermalink: /fastcore/\nsearch: 'false'\ntitle: 'fastcore: An Underrated Python Library'\ntoc: false\n\n---\n\n\n---\naliases:\n- /fastcore/\nauthor: Hamel Husain\nbadges: true\ncategories:\n- fastcore\n- fastai\ndate: '2020-09-01'\ndescription: A unique python library that extends the python programming language\n and provides utilities that enhance productivity.\ndraft: 'true'\nimage: fastcore_imgs/td.png\noutput-file: 2020-09-01-fastcore.html\npermalink: /fastcore/\nsearch: 'false'\ntitle: 'fastcore: An Underrated Python Library'\ntoc: false\n\n---\n\n",
+ "output_type": "error",
+ "traceback": [
+ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+ "\u001b[0;31mAssertionError\u001b[0m Traceback (most recent call last)",
+ "Cell \u001b[0;32mIn[30], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m#|hide\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m \u001b[43mtest_eq\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_fm1\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\"\"\u001b[39;49m\u001b[38;5;124;43m---\u001b[39;49m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;124;43maliases:\u001b[39;49m\n\u001b[1;32m 4\u001b[0m \u001b[38;5;124;43m- /fastcore/\u001b[39;49m\n\u001b[1;32m 5\u001b[0m \u001b[38;5;124;43mauthor: Hamel Husain\u001b[39;49m\n\u001b[1;32m 6\u001b[0m \u001b[38;5;124;43mbadges: true\u001b[39;49m\n\u001b[1;32m 7\u001b[0m \u001b[38;5;124;43mcategories:\u001b[39;49m\n\u001b[1;32m 8\u001b[0m \u001b[38;5;124;43m- fastcore\u001b[39;49m\n\u001b[1;32m 9\u001b[0m \u001b[38;5;124;43m- fastai\u001b[39;49m\n\u001b[1;32m 10\u001b[0m \u001b[38;5;124;43mdate: \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m2020-09-01\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\n\u001b[1;32m 11\u001b[0m \u001b[38;5;124;43mdescription: A unique python library that extends the python programming language\u001b[39;49m\n\u001b[1;32m 12\u001b[0m \u001b[38;5;124;43m and provides utilities that enhance productivity.\u001b[39;49m\n\u001b[1;32m 13\u001b[0m \u001b[38;5;124;43mdraft: \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mtrue\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\n\u001b[1;32m 14\u001b[0m \u001b[38;5;124;43mimage: fastcore_imgs/td.png\u001b[39;49m\n\u001b[1;32m 15\u001b[0m \u001b[38;5;124;43moutput-file: 2020-09-01-fastcore.html\u001b[39;49m\n\u001b[1;32m 16\u001b[0m \u001b[38;5;124;43mpermalink: /fastcore/\u001b[39;49m\n\u001b[1;32m 17\u001b[0m \u001b[38;5;124;43msearch: \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mfalse\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\n\u001b[1;32m 18\u001b[0m \u001b[38;5;124;43mtitle: \u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mfastcore: An Underrated Python Library\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\n\u001b[1;32m 19\u001b[0m \u001b[38;5;124;43mtoc: false\u001b[39;49m\n\u001b[1;32m 20\u001b[0m \n\u001b[1;32m 21\u001b[0m \u001b[38;5;124;43m---\u001b[39;49m\n\u001b[1;32m 22\u001b[0m \n\u001b[1;32m 23\u001b[0m \u001b[38;5;124;43m\"\"\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m 26\u001b[0m _res\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\"\"\u001b[39m\u001b[38;5;124m---\u001b[39m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;124maliases:\u001b[39m\n\u001b[1;32m 28\u001b[0m \u001b[38;5;124m- /jupyter/2020/02/20/test\u001b[39m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 40\u001b[0m \n\u001b[1;32m 41\u001b[0m \u001b[38;5;124m\"\"\"\u001b[39m\n\u001b[1;32m 43\u001b[0m nbp \u001b[38;5;241m=\u001b[39m NBProcessor(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m../../tests/2020-02-20-test.ipynb\u001b[39m\u001b[38;5;124m'\u001b[39m, procs\u001b[38;5;241m=\u001b[39m[FrontmatterProc, MigrateProc])\n",
+ "File \u001b[0;32m~/miniconda3/envs/nbdev/lib/python3.10/site-packages/fastcore/test.py:39\u001b[0m, in \u001b[0;36mtest_eq\u001b[0;34m(a, b)\u001b[0m\n\u001b[1;32m 37\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21mtest_eq\u001b[39m(a,b):\n\u001b[1;32m 38\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`test` that `a==b`\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m---> 39\u001b[0m \u001b[43mtest\u001b[49m\u001b[43m(\u001b[49m\u001b[43ma\u001b[49m\u001b[43m,\u001b[49m\u001b[43mb\u001b[49m\u001b[43m,\u001b[49m\u001b[43mequals\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcname\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43m==\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n",
+ "File \u001b[0;32m~/miniconda3/envs/nbdev/lib/python3.10/site-packages/fastcore/test.py:29\u001b[0m, in \u001b[0;36mtest\u001b[0;34m(a, b, cmp, cname)\u001b[0m\n\u001b[1;32m 27\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m`assert` that `cmp(a,b)`; display inputs and `cname or cmp.__name__` if it fails\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 28\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m cname \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m: cname\u001b[38;5;241m=\u001b[39mcmp\u001b[38;5;241m.\u001b[39m\u001b[38;5;18m__name__\u001b[39m\n\u001b[0;32m---> 29\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m cmp(a,b),\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00ma\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;132;01m{\u001b[39;00mb\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n",
+ "\u001b[0;31mAssertionError\u001b[0m: ==:\n---\naliases:\n- /fastcore/\nauthor: Hamel Husain\ncategories:\n- fastcore\n- fastai\ndate: '2020-09-01'\ndescription: A unique python library that extends the python programming language\n and provides utilities that enhance productivity.\ndraft: 'true'\nimage: fastcore_imgs/td.png\noutput-file: 2020-09-01-fastcore.html\npermalink: /fastcore/\nsearch: 'false'\ntitle: 'fastcore: An Underrated Python Library'\ntoc: false\n\n---\n\n\n---\naliases:\n- /fastcore/\nauthor: Hamel Husain\nbadges: true\ncategories:\n- fastcore\n- fastai\ndate: '2020-09-01'\ndescription: A unique python library that extends the python programming language\n and provides utilities that enhance productivity.\ndraft: 'true'\nimage: fastcore_imgs/td.png\noutput-file: 2020-09-01-fastcore.html\npermalink: /fastcore/\nsearch: 'false'\ntitle: 'fastcore: An Underrated Python Library'\ntoc: false\n\n---\n\n"
+ ]
+ }
+ ],
"source": [
"#|hide\n",
"test_eq(_fm1, \"\"\"---\n",
diff --git a/tests/docs_test.ipynb b/tests/docs_test.ipynb
index 273db4d42..ca4d868e4 100644
--- a/tests/docs_test.ipynb
+++ b/tests/docs_test.ipynb
@@ -20,7 +20,30 @@
"> A description\n",
"- key1: value1\n",
"- key2: value2\n",
- "- categories: [c1, c2]"
+ "- categories: [c1, c2]\n",
+ "\n",
+ "\n",
+ "With some remaining text that I want to preserve!!\n",
+ "\n",
+ "E.g., see this list:\n",
+ "\n",
+ "- [With a link](https://example.com)\n",
+ "\n",
+ "And this table:\n",
+ "\n",
+ "| Column 1 | Column 2 |\n",
+ "|----------|----------|\n",
+ "\n",
+ "and this example code:\n",
+ "\n",
+ "```python\n",
+ "a = 4 + 6\n",
+ "```\n",
+ "\n",
+ "And this image:\n",
+ "\n",
+ "\n",
+ "\n"
]
},
{
From 04c87d586a9fbb66c2bbb23dc33d555d73eaa90a Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Sun, 1 Jun 2025 23:09:28 -0400
Subject: [PATCH 23/31] Cleanup frontmatter code
---
nbdev/frontmatter.py | 40 +++++++++---------------------------
nbs/api/09_frontmatter.ipynb | 40 +++++++++---------------------------
2 files changed, 20 insertions(+), 60 deletions(-)
diff --git a/nbdev/frontmatter.py b/nbdev/frontmatter.py
index 97dfa5ffc..8ee55060b 100644
--- a/nbdev/frontmatter.py
+++ b/nbdev/frontmatter.py
@@ -22,32 +22,20 @@
_re_fm_nb = re.compile(_RE_FM_BASE+'$', flags=re.DOTALL)
_re_fm_md = re.compile(_RE_FM_BASE, flags=re.DOTALL)
-
-# Regex 1: Capture title and description only
_re_fm_title_desc = re.compile(r'^#\s+(\S.*?)(?:\n|$)(?:\s*\n)*(?:>\s+(\S.*?)(?:\n|$)(?:\s*\n)*)?', flags=re.MULTILINE)
-
-# Regex 2: Capture contiguous KV block (lines starting with - or blank lines)
_re_fm_kv = re.compile(r'^((?:\s*-\s+[a-zA-Z_][a-zA-Z0-9_]*\s*:\s+.*(?:\n|$)|\s*\n)*)', flags=re.MULTILINE)
def _parse_kv_block(block_text):
"""Parse a block of key-value pairs from lines starting with '-'"""
- if not block_text.strip():
- return {}
-
- # Extract only the lines that start with '-' (ignore blank lines)
+ if not block_text.strip(): return {}
kv_lines = []
for line in block_text.split('\n'):
line = line.strip()
if line.startswith('-'):
kv_content = line[1:].strip() # Remove the '-' and strip whitespace
- if kv_content:
- kv_lines.append(kv_content)
-
- if not kv_lines:
- return {}
-
- try:
- return yaml.safe_load('\n'.join(kv_lines))
+ if kv_content: kv_lines.append(kv_content)
+ if not kv_lines: return {}
+ try: return yaml.safe_load('\n'.join(kv_lines))
except Exception as e:
warn(f'Failed to create YAML dict for:\n{kv_lines}\n\n{e}\n')
return {}
@@ -55,22 +43,17 @@ def _parse_kv_block(block_text):
def _md2dict(s:str):
"Convert H1 formatted markdown cell to frontmatter dict"
if '#' not in s: return {}
-
res = {}
remaining_start = 0
- # Step 1: Extract title and description
+ # Extract title and description
title_desc_match = _re_fm_title_desc.match(s)
- if not title_desc_match:
- return {}
-
+ if not title_desc_match: return {}
res['title'] = title_desc_match.group(1)
- if title_desc_match.group(2):
- res['description'] = title_desc_match.group(2)
-
+ if title_desc_match.group(2): res['description'] = title_desc_match.group(2)
remaining_start = title_desc_match.end()
- # Step 2: Extract KV pairs starting from where title/desc ended
+ # Extract KV pairs starting from where title/desc ended
kv_text = s[remaining_start:]
kv_match = _re_fm_kv.match(kv_text)
if kv_match:
@@ -78,12 +61,9 @@ def _md2dict(s:str):
res.update(kv_dict)
remaining_start += kv_match.end()
- # Step 3: Everything else is remaining content
+ # Extract remaining content
remaining = s[remaining_start:].strip()
- if remaining:
- # print("REMAINING ", remaining)
- # print("Total string", s)
- res['__remaining'] = remaining
+ if remaining: res['__remaining'] = remaining
return res
diff --git a/nbs/api/09_frontmatter.ipynb b/nbs/api/09_frontmatter.ipynb
index 6acb2060c..463fffe4f 100644
--- a/nbs/api/09_frontmatter.ipynb
+++ b/nbs/api/09_frontmatter.ipynb
@@ -75,32 +75,20 @@
"_re_fm_nb = re.compile(_RE_FM_BASE+'$', flags=re.DOTALL)\n",
"_re_fm_md = re.compile(_RE_FM_BASE, flags=re.DOTALL)\n",
"\n",
- "\n",
- "# Regex 1: Capture title and description only\n",
"_re_fm_title_desc = re.compile(r'^#\\s+(\\S.*?)(?:\\n|$)(?:\\s*\\n)*(?:>\\s+(\\S.*?)(?:\\n|$)(?:\\s*\\n)*)?', flags=re.MULTILINE)\n",
- "\n",
- "# Regex 2: Capture contiguous KV block (lines starting with - or blank lines)\n",
"_re_fm_kv = re.compile(r'^((?:\\s*-\\s+[a-zA-Z_][a-zA-Z0-9_]*\\s*:\\s+.*(?:\\n|$)|\\s*\\n)*)', flags=re.MULTILINE)\n",
"\n",
"def _parse_kv_block(block_text):\n",
" \"\"\"Parse a block of key-value pairs from lines starting with '-'\"\"\"\n",
- " if not block_text.strip():\n",
- " return {}\n",
- " \n",
- " # Extract only the lines that start with '-' (ignore blank lines)\n",
+ " if not block_text.strip(): return {}\n",
" kv_lines = []\n",
" for line in block_text.split('\\n'):\n",
" line = line.strip()\n",
" if line.startswith('-'):\n",
" kv_content = line[1:].strip() # Remove the '-' and strip whitespace\n",
- " if kv_content:\n",
- " kv_lines.append(kv_content)\n",
- " \n",
- " if not kv_lines:\n",
- " return {}\n",
- " \n",
- " try:\n",
- " return yaml.safe_load('\\n'.join(kv_lines))\n",
+ " if kv_content: kv_lines.append(kv_content)\n",
+ " if not kv_lines: return {}\n",
+ " try: return yaml.safe_load('\\n'.join(kv_lines))\n",
" except Exception as e:\n",
" warn(f'Failed to create YAML dict for:\\n{kv_lines}\\n\\n{e}\\n')\n",
" return {}\n",
@@ -108,22 +96,17 @@
"def _md2dict(s:str):\n",
" \"Convert H1 formatted markdown cell to frontmatter dict\"\n",
" if '#' not in s: return {}\n",
- " \n",
" res = {}\n",
" remaining_start = 0\n",
" \n",
- " # Step 1: Extract title and description\n",
+ " # Extract title and description\n",
" title_desc_match = _re_fm_title_desc.match(s)\n",
- " if not title_desc_match:\n",
- " return {}\n",
- " \n",
+ " if not title_desc_match: return {}\n",
" res['title'] = title_desc_match.group(1)\n",
- " if title_desc_match.group(2):\n",
- " res['description'] = title_desc_match.group(2)\n",
- " \n",
+ " if title_desc_match.group(2): res['description'] = title_desc_match.group(2)\n",
" remaining_start = title_desc_match.end()\n",
" \n",
- " # Step 2: Extract KV pairs starting from where title/desc ended\n",
+ " # Extract KV pairs starting from where title/desc ended\n",
" kv_text = s[remaining_start:]\n",
" kv_match = _re_fm_kv.match(kv_text)\n",
" if kv_match:\n",
@@ -131,12 +114,9 @@
" res.update(kv_dict)\n",
" remaining_start += kv_match.end()\n",
" \n",
- " # Step 3: Everything else is remaining content\n",
+ " # Extract remaining content\n",
" remaining = s[remaining_start:].strip()\n",
- " if remaining:\n",
- " # print(\"REMAINING \", remaining)\n",
- " # print(\"Total string\", s)\n",
- " res['__remaining'] = remaining\n",
+ " if remaining: res['__remaining'] = remaining\n",
" \n",
" return res\n",
"\n",
From deebd534001477d251aa027e9f7f5c29e1233f9c Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:21:03 -0400
Subject: [PATCH 24/31] Copy .qmd for docs
---
nbdev/serve.py | 4 +++-
nbs/api/17_serve.ipynb | 4 +++-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/nbdev/serve.py b/nbdev/serve.py
index c9fce81e2..d1ed2c442 100644
--- a/nbdev/serve.py
+++ b/nbdev/serve.py
@@ -46,7 +46,9 @@ def _proc_file(s, cache, path, mtime=None):
if s.stat().st_mtime<=dtime: return
d.parent.mkdir(parents=True, exist_ok=True)
- if s.suffix in ['.ipynb','.qmd']: return s,d,FilterDefaults
+ if s.suffix in ['.ipynb','.qmd']:
+ if s.suffix == '.qmd': copy2(s,d) # still need .qmd files for quarto links to work
+ return s,d,FilterDefaults
md = _is_qpy(s)
if md is not None: return s,d,md.strip()
else: copy2(s,d)
diff --git a/nbs/api/17_serve.ipynb b/nbs/api/17_serve.ipynb
index 8198c6dc1..829866aec 100644
--- a/nbs/api/17_serve.ipynb
+++ b/nbs/api/17_serve.ipynb
@@ -96,7 +96,9 @@
" if s.stat().st_mtime<=dtime: return\n",
"\n",
" d.parent.mkdir(parents=True, exist_ok=True)\n",
- " if s.suffix in ['.ipynb','.qmd']: return s,d,FilterDefaults\n",
+ " if s.suffix in ['.ipynb','.qmd']: \n",
+ " if s.suffix == '.qmd': copy2(s,d) # still need .qmd files for quarto links to work\n",
+ " return s,d,FilterDefaults\n",
" md = _is_qpy(s)\n",
" if md is not None: return s,d,md.strip()\n",
" else: copy2(s,d)"
From f49bd4bbf78ccf10586455fccd88ec9885f11181 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Mon, 2 Jun 2025 16:42:07 -0400
Subject: [PATCH 25/31] Update nbs
---
convert_nbs.py | 98 --------------------------------------------------
1 file changed, 98 deletions(-)
delete mode 100644 convert_nbs.py
diff --git a/convert_nbs.py b/convert_nbs.py
deleted file mode 100644
index bf26a031c..000000000
--- a/convert_nbs.py
+++ /dev/null
@@ -1,98 +0,0 @@
-#!/usr/bin/env python
-
-import os
-import shutil
-import re
-from pathlib import Path
-from execnb.nbio import read_nb, dict2nb
-from nbdev.process import read_qmd, write_qmd, read_nb_or_qmd
-from fastcore.script import call_parse
-
-
-@call_parse
-def convert_notebooks(
- source_folder: str, # Source folder containing .ipynb files
- dest_folder: str # Destination folder for .qmd files and copied items
-):
- """
- Converts .ipynb files from source_folder to .qmd files in dest_folder.
- Other files are copied directly. Replicates directory structure.
- """
- source_dir = Path(source_folder)
- dest_dir = Path(dest_folder)
-
- if not source_dir.is_dir():
- print(f"Error: Source directory '{source_dir.resolve()}' does not exist or is not a directory.")
- return
-
- print(f"Source directory: {source_dir.resolve()}")
- print(f"Destination directory: {dest_dir.resolve()}")
-
- dest_dir.mkdir(parents=True, exist_ok=True)
- print(f"Ensured destination directory exists: {dest_dir}")
-
- total_files_processed = 0
- notebooks_converted = 0
- files_copied = 0
- errors_encountered = 0
-
- for root, _, files in os.walk(source_dir):
- current_source_dir = Path(root)
- # Calculate path relative to the initial source_dir
- # to replicate the structure in dest_dir
- relative_subdir_path = current_source_dir.relative_to(source_dir)
- current_dest_dir = dest_dir / relative_subdir_path
-
- # Ensure the subdirectory structure exists in the destination
- # (os.walk guarantees `root` exists, so mkdir for current_dest_dir is usually fine)
- current_dest_dir.mkdir(parents=True, exist_ok=True)
-
- for filename in files:
- total_files_processed += 1
- source_file_path = current_source_dir / filename
-
- if source_file_path.name.startswith('.'): # Skip hidden files like .DS_Store
- print(f"Skipping hidden file: {source_file_path}")
- # Decrement count as we are not "processing" it in terms of conversion/copy
- total_files_processed -=1
- continue
-
- if source_file_path.is_dir(): # Should not happen with os.walk's `files` list, but as a safeguard
- print(f"Skipping directory listed as file: {source_file_path}")
- total_files_processed -=1
- continue
-
- if source_file_path.suffix == ".ipynb":
- # Prepare destination path for .qmd file
- dest_qmd_filename = source_file_path.stem + ".qmd"
- dest_file_path = current_dest_dir / dest_qmd_filename
-
- print(f"Processing for conversion: {source_file_path} -> {dest_file_path}")
- try:
- # For .ipynb, read_nb_or_qmd will use read_nb from execnb
- notebook_object = read_nb_or_qmd(source_file_path)
- write_qmd(notebook_object, dest_file_path)
- print(f" Successfully converted '{source_file_path.name}' to '{dest_file_path.name}'")
- notebooks_converted +=1
- except Exception as e:
- print(f" Error converting {source_file_path}: {e}")
- errors_encountered +=1
- else:
- # For any other file type, copy it directly
- dest_file_path = current_dest_dir / filename
- print(f"Copying: {source_file_path} -> {dest_file_path}")
- try:
- shutil.copy2(source_file_path, dest_file_path) # copy2 preserves metadata
- print(f" Successfully copied '{source_file_path.name}'")
- files_copied += 1
- except Exception as e:
- print(f" Error copying {source_file_path}: {e}")
- errors_encountered +=1
-
- print(f"\n--- Conversion Summary ---")
- print(f"Total items scanned in source: {total_files_processed}")
- print(f"Notebooks converted to .qmd: {notebooks_converted}")
- print(f"Other files copied: {files_copied}")
- if errors_encountered > 0:
- print(f"Errors encountered: {errors_encountered}")
- print(f"Output located in: {dest_dir.resolve()}")
\ No newline at end of file
From 90db296506db8158facda6e2207391b52d7bb20d Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Mon, 2 Jun 2025 17:11:17 -0400
Subject: [PATCH 26/31] Allow frontmatter keys to contain hyphens
---
nbdev/frontmatter.py | 2 +-
nbs/api/09_frontmatter.ipynb | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/nbdev/frontmatter.py b/nbdev/frontmatter.py
index 8ee55060b..cafaa95dd 100644
--- a/nbdev/frontmatter.py
+++ b/nbdev/frontmatter.py
@@ -23,7 +23,7 @@
_re_fm_md = re.compile(_RE_FM_BASE, flags=re.DOTALL)
_re_fm_title_desc = re.compile(r'^#\s+(\S.*?)(?:\n|$)(?:\s*\n)*(?:>\s+(\S.*?)(?:\n|$)(?:\s*\n)*)?', flags=re.MULTILINE)
-_re_fm_kv = re.compile(r'^((?:\s*-\s+[a-zA-Z_][a-zA-Z0-9_]*\s*:\s+.*(?:\n|$)|\s*\n)*)', flags=re.MULTILINE)
+_re_fm_kv = re.compile(r'^((?:\s*-\s+[a-zA-Z_][a-zA-Z0-9_-]*\s*:\s+.*(?:\n|$)|\s*\n)*)', flags=re.MULTILINE)
def _parse_kv_block(block_text):
"""Parse a block of key-value pairs from lines starting with '-'"""
diff --git a/nbs/api/09_frontmatter.ipynb b/nbs/api/09_frontmatter.ipynb
index 463fffe4f..cddf629a2 100644
--- a/nbs/api/09_frontmatter.ipynb
+++ b/nbs/api/09_frontmatter.ipynb
@@ -76,7 +76,7 @@
"_re_fm_md = re.compile(_RE_FM_BASE, flags=re.DOTALL)\n",
"\n",
"_re_fm_title_desc = re.compile(r'^#\\s+(\\S.*?)(?:\\n|$)(?:\\s*\\n)*(?:>\\s+(\\S.*?)(?:\\n|$)(?:\\s*\\n)*)?', flags=re.MULTILINE)\n",
- "_re_fm_kv = re.compile(r'^((?:\\s*-\\s+[a-zA-Z_][a-zA-Z0-9_]*\\s*:\\s+.*(?:\\n|$)|\\s*\\n)*)', flags=re.MULTILINE)\n",
+ "_re_fm_kv = re.compile(r'^((?:\\s*-\\s+[a-zA-Z_][a-zA-Z0-9_-]*\\s*:\\s+.*(?:\\n|$)|\\s*\\n)*)', flags=re.MULTILINE)\n",
"\n",
"def _parse_kv_block(block_text):\n",
" \"\"\"Parse a block of key-value pairs from lines starting with '-'\"\"\"\n",
From c503fc4fab0e1bf27925c6e484528a560b0cd1bc Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Mon, 2 Jun 2025 17:53:46 -0400
Subject: [PATCH 27/31] Fix index.qmd not found
---
nbdev/serve.py | 2 +-
nbs/api/17_serve.ipynb | 13 +++++++------
2 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/nbdev/serve.py b/nbdev/serve.py
index d1ed2c442..39852e4a6 100644
--- a/nbdev/serve.py
+++ b/nbdev/serve.py
@@ -47,7 +47,7 @@ def _proc_file(s, cache, path, mtime=None):
d.parent.mkdir(parents=True, exist_ok=True)
if s.suffix in ['.ipynb','.qmd']:
- if s.suffix == '.qmd': copy2(s,d) # still need .qmd files for quarto links to work
+ if s.name == 'index.qmd': copy2(s,d)
return s,d,FilterDefaults
md = _is_qpy(s)
if md is not None: return s,d,md.strip()
diff --git a/nbs/api/17_serve.ipynb b/nbs/api/17_serve.ipynb
index 829866aec..af9693716 100644
--- a/nbs/api/17_serve.ipynb
+++ b/nbs/api/17_serve.ipynb
@@ -97,7 +97,7 @@
"\n",
" d.parent.mkdir(parents=True, exist_ok=True)\n",
" if s.suffix in ['.ipynb','.qmd']: \n",
- " if s.suffix == '.qmd': copy2(s,d) # still need .qmd files for quarto links to work\n",
+ " if s.name == 'index.qmd': copy2(s,d)\n",
" return s,d,FilterDefaults\n",
" md = _is_qpy(s)\n",
" if md is not None: return s,d,md.strip()\n",
@@ -113,11 +113,12 @@
"source": [
"#|hide\n",
"# __file__ = '../tutorials/circles.svg.py'\n",
- "# p = Path(__file__).resolve()\n",
- "# cfg = get_config()\n",
- "# cache = cfg.config_path/'_proc'\n",
- "# path = Path(cfg.nbs_path)\n",
- "# _proc_file(p, cache, path)"
+ "__file__ = '../blog/index.qmd'\n",
+ "p = Path(__file__).resolve()\n",
+ "cfg = get_config()\n",
+ "cache = cfg.config_path/'_proc'\n",
+ "path = Path(cfg.nbs_path)\n",
+ "_proc_file(p, cache, path)"
]
},
{
From cb5a4102c851f906ede55d106c05540ea5ecf61f Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Tue, 3 Jun 2025 16:21:45 -0400
Subject: [PATCH 28/31] Initial draft of .qmd file tutorial
---
nbs/tutorials/develop_in_plain_text.qmd | 152 ++++++++++++++++++++++++
1 file changed, 152 insertions(+)
create mode 100644 nbs/tutorials/develop_in_plain_text.qmd
diff --git a/nbs/tutorials/develop_in_plain_text.qmd b/nbs/tutorials/develop_in_plain_text.qmd
new file mode 100644
index 000000000..3e7fe4e26
--- /dev/null
+++ b/nbs/tutorials/develop_in_plain_text.qmd
@@ -0,0 +1,152 @@
+# Develop in plain text
+> Avoid the overhead of developing in .ipynb notebooks while working in plain text.
+
+Using [quarto](https://quarto.org/docs/get-started/hello/vscode.html) and its [VS code extension](https://marketplace.visualstudio.com/items?itemName=quarto.quarto), it is possible to use `.qmd` files instead of `.ipynb` files for a interactive, plain text library development experience using `nbdev`. That `.qmd` files are plain text comes with several advantages:
+
+1. 🤖 `.qmd` **seamlessly integrates with AI copilots** (e.g., Cursor) with no overhead of a special notebook extension
+2. 🔄 `.qmd` is **fully compatible with standard git tooling**. No need for special notebook diff tools
+3. 🪄 `.qmd` **works with your favorite editing style** (e.g., VIM)
+4. 🫧 `.qmd` **source files stay clean** during `nbdev`'s transpilation process. You don't need a special `nbdev_clean` step to remove cell metadata and outputs.
+
+All this while remaining fully interactive thanks to VSCode's [interactive window](https://code.visualstudio.com/docs/python/jupyter-support-py) (see [below](#example))
+
+Starting from version YYY, `nbdev` will automatically support both `.qmd` files and `.ipynb` files for authoring the main library code+docs+tests.
+
+:::{.callout-warning}
+Ensure that all files under your `nbs/` directory have distinct names. E.g., do not have both `00_core.ipynb` and `00_core.qmd`, as both of these will create the intermediate `_proc/00_core.ipynb`
+:::
+
+## Interactively develop in .qmd {#example}
+
+1. Install the [quarto VSCode extension](https://marketplace.visualstudio.com/items?itemName=quarto.quarto)
+2. Open a `.qmd` file in VSCode.
+
+
+## Helpful hotkeys
+
+Below are some helpful quarto hotkeys that have no keyboard shortcuts by default. Feel free to adjust the keybindings to your liking.
+
+| Command | Hotkey |
+|--------|---------|
+| "editor.action.showDefinitionPreviewHover" | `shift+tab` |
+| "quarto.insertCodeCell" | `cmd+shift+;` |
+| "jupyter.execSelectionInteractive" | `cmd+' cmd+'` |
+| "quarto.runCellsAbove" | `shift+cmd+,` |
+| "quarto.runCellsBelow" | `shift+cmd+.` |
+| "quarto.runAllCells" | `cmd+shift+'` |
+
+::: {.callout-note collapse="true"}
+## Or copy the following json to your user settings
+Use `Cmd+Shift+P` > "Preferences: Open User Settings (JSON)" to open your user settings. Then copy and paste the following json:
+
+```js
+ {
+ "key": "shift+tab",
+ "command": "editor.action.showDefinitionPreviewHover"
+ },
+ {
+ "key": "shift+cmd+;",
+ "command": "quarto.insertCodeCell",
+ "when": "editorTextFocus && !findInputFocussed && !replaceInputFocussed && editorLangId == 'quarto'"
+ },
+ {
+ "key": "cmd+' cmd+'",
+ "command": "jupyter.execSelectionInteractive",
+ "when": "editorTextFocus && isWorkspaceTrusted && !findInputFocussed && !notebookEditorFocused && !replaceInputFocussed && (editorLangId == 'markdown' || editorLangId == 'quarto' || editorLangId == 'python')"
+ },
+ {
+ "key": "shift+cmd+,",
+ "command": "quarto.runCellsAbove",
+ "when": "editorTextFocus && !findInputFocussed && !replaceInputFocussed && editorLangId == 'quarto'"
+ },
+ {
+ "key": "shift+cmd+,",
+ "command": "quarto.runCellsAbove",
+ "when": "activeCustomEditorId == 'quarto.visualEditor'"
+ },
+ {
+ "key": "shift+cmd+.",
+ "command": "quarto.runCellsBelow",
+ "when": "editorTextFocus && !findInputFocussed && !replaceInputFocussed && editorLangId == 'quarto'"
+ },
+ {
+ "key": "shift+cmd+.",
+ "command": "quarto.runCellsBelow",
+ "when": "activeCustomEditorId == 'quarto.visualEditor'"
+ },
+ {
+ "key": "shift+cmd+'",
+ "command": "quarto.runAllCells",
+ "when": "editorTextFocus && !findInputFocussed && !replaceInputFocussed && editorLangId == 'quarto'"
+ },
+ {
+ "key": "shift+cmd+'",
+ "command": "quarto.runAllCells",
+ "when": "activeCustomEditorId == 'quarto.visualEditor'"
+ },
+```
+
+:::
+
+## Helpful VSCode settings
+Open your VSCode settings (`Cmd+Shift+P` > "Preferences: Open User Settings (JSON)") and enable the following:
+
+- **Enable `Interactive Window: Execute with Shift+Enter`** --- Run code like you would with jupyter
+- **Set `Jupyter > Interactive Window: Creation Mode` to `perFile`** --- Each file has its own dedicated kernel, just like in jupyter
+- **Enable `Jupyter > Interactive Window > Text Editor: Magic commands as comments`** --- When checked, lets us use jupyter magics for cells made in plain text files (otherwise the `%` symbols break the interactive python)
+
+:::{.callout-note collapse="true"}
+## Or copy and paste the following into your user settings
+```js
+//...
+ "interactiveWindow.executeWithShiftEnter": true,
+ "jupyter.interactiveWindow.creationMode": "perFile",
+ "jupyter.interactiveWindow.textEditor.magicCommandsAsComments": true,
+ "jupyter.interactiveWindow.textEditor.autoAddNewCell": false
+//...
+```
+:::
+
+
+
+## Run `nbdev_export` in VSCode on file save
+
+> Never let your `.qmd` source get out of sync with your `.py` library.
+
+Automatically run `nbdev_export` on save to keep your `.qmd` source in sync with your `.py` library.
+
+1. Install the [`Run on Save`](https://marketplace.visualstudio.com/items?itemName=emeraldwalk.RunOnSave) VSCode extension
+2. Copy and paste the following into your user/workspace settings (`Cmd+Shift+P` then either "Preferences: Open User settings (JSON)" or "Preferences: Open Workspace settings (JSON)")
+
+```js
+{
+ "files.watcherExclude": {
+ "**/.git/objects/**": true,
+ "**/.git/subtree-cache/**": true,
+ "**/node_modules/*/**": true,
+ "**/.hg/store/**": true,
+ },
+ "emeraldwalk.runonsave": {
+ "commands": [
+ {
+ "match": "nbs/.*\\.qmd$", // Replace with your own nbs/ directory
+ "cmd": "source ${workspaceFolder}/.venv/bin/activate && nbdev_export", // Replace with a path to your python env where `nbdev` is installed
+ }
+ ]
+ }
+}
+```
+
+Now whenever you save a `.qmd` file, `nbdev_export` will automatically compile your source code for you (this approach will also work for .ipynb files if you update the `match` pattern to `nbs/.*\\.ipynb$`).
+
+You will still need to run `nbdev_prepare` to test the notebooks and generate the docs.
+
+## Convert existing .ipynb codebase to .qmd
+
+So you have an existing `.ipynb` codebase written in the `nbs/` directory that you want to convert to `.qmd` files.
+
+Converting to `.qmd` is easy:
+
+1. Run `nbdev_ipynb_to_qmd nbs nbs_qmd && mv nbs nbs_ipynb && mv nbs_qmd nbs` to convert your `.ipynb` files to `.qmd` files (feel free to `rm -r nbs_ipynb`)
+2. Update the `readme_nb` in `settings.ini` to be the new `.qmd` file
+3. Run `nbdev_prepare`
\ No newline at end of file
From f27e408e5786d6e8169eda2813070332560f5e0d Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Tue, 3 Jun 2025 16:35:50 -0400
Subject: [PATCH 29/31] Add annotations to custom functions
---
nbdev/frontmatter.py | 4 ++--
nbdev/qmd.py | 24 +++++++++++-------------
nbs/api/09_frontmatter.ipynb | 4 ++--
nbs/api/15_qmd.ipynb | 24 +++++++++++-------------
4 files changed, 26 insertions(+), 30 deletions(-)
diff --git a/nbdev/frontmatter.py b/nbdev/frontmatter.py
index cafaa95dd..332e05746 100644
--- a/nbdev/frontmatter.py
+++ b/nbdev/frontmatter.py
@@ -25,7 +25,7 @@
_re_fm_title_desc = re.compile(r'^#\s+(\S.*?)(?:\n|$)(?:\s*\n)*(?:>\s+(\S.*?)(?:\n|$)(?:\s*\n)*)?', flags=re.MULTILINE)
_re_fm_kv = re.compile(r'^((?:\s*-\s+[a-zA-Z_][a-zA-Z0-9_-]*\s*:\s+.*(?:\n|$)|\s*\n)*)', flags=re.MULTILINE)
-def _parse_kv_block(block_text):
+def _parse_kv_block(block_text: str):
"""Parse a block of key-value pairs from lines starting with '-'"""
if not block_text.strip(): return {}
kv_lines = []
@@ -41,7 +41,7 @@ def _parse_kv_block(block_text):
return {}
def _md2dict(s:str):
- "Convert H1 formatted markdown cell to frontmatter dict"
+ "Convert custom H1 formatted markdown cell to frontmatter dict"
if '#' not in s: return {}
res = {}
remaining_start = 0
diff --git a/nbdev/qmd.py b/nbdev/qmd.py
index cada59135..b419828a5 100644
--- a/nbdev/qmd.py
+++ b/nbdev/qmd.py
@@ -20,22 +20,20 @@
'ipynb_to_qmd']
# %% ../nbs/api/15_qmd.ipynb
-def _qmd_to_raw_cell(source_str, cell_type_str, qmd_metadata=None):
+def _qmd_to_raw_cell(source:str, cell_type:str, qmd_metadata=None):
"""Create a default ipynb json cell"""
- cell = {'cell_type': cell_type_str, 'metadata': {}, 'source': source_str}
- if cell_type_str == 'code':
+ cell = {'cell_type': cell_type, 'metadata': {}, 'source': source}
+ if cell_type == 'code':
cell['execution_count'] = None
cell['outputs'] = []
if qmd_metadata: cell['qmd_metadata'] = qmd_metadata
return cell
-def read_qmd(path):
+def read_qmd(path:str):
"""Reads a .qmd file as an nb compatible with the rest of execnb and nbdev"""
content = Path(path).read_text(encoding='utf-8')
- # Modified regex to capture the metadata between {python and }
cell_pat = re.compile(r"^(`{3,})\s*\{python([^\}]*)\}\s*\n(.*?)^\1\s*$", re.MULTILINE | re.DOTALL)
- # `parts` will be [md_chunk, captured_backticks_1, captured_metadata_1, captured_code_1, md_chunk_2, ...]
parts = cell_pat.split(content)
raw_cells = []
@@ -45,11 +43,11 @@ def read_qmd(path):
# 4 items per match: [md, backticks, metadata?, code]
for i in range(1, len(parts), 4):
- if i + 2 < len(parts): # We have backticks, metadata, and code
+ if i + 2 < len(parts): # backticks, metadata, code
metadata = parts[i+1] # The captured metadata
code_source = parts[i+2].strip() # The captured code
if code_source: raw_cells.append(_qmd_to_raw_cell(code_source, 'code', metadata if metadata else None))
- if i + 3 < len(parts): # Intermediate markdown
+ if i + 3 < len(parts): # intermediate markdown
intermediate_md_source = parts[i+3].strip()
if intermediate_md_source: raw_cells.append(_qmd_to_raw_cell(intermediate_md_source, 'markdown'))
@@ -68,13 +66,13 @@ def read_qmd(path):
return dict2nb(notebook_dict)
-def read_nb_or_qmd(path):
+def read_nb_or_qmd(path:str):
if Path(path).suffix == '.qmd': return read_qmd(path)
return read_nb(path)
# %% ../nbs/api/15_qmd.ipynb
-def _get_fence_ticks(source):
+def _get_fence_ticks(source:str):
"""Determine the number of backticks needed for fencing that won't conflict with source"""
if '`' not in source: return '```' # Default to 3 if no backticks in source
@@ -88,7 +86,7 @@ def _get_fence_ticks(source):
while num_ticks in used_lengths: num_ticks += 1
return '`' * num_ticks
-def _nb_to_qmd_str(nb):
+def _nb_to_qmd_str(nb: AttrDict):
"""Convert a notebook to a string in .qmd format"""
def cell_to_qmd(cell):
source = cell.source.rstrip('\n')
@@ -101,12 +99,12 @@ def cell_to_qmd(cell):
return ''
return '\n\n'.join(filter(None, [cell_to_qmd(cell) for cell in nb.cells]))
-def write_qmd(nb, path):
+def write_qmd(nb: AttrDict, path:str):
"""Write a notebook back to .qmd format"""
qmd_str = _nb_to_qmd_str(nb)
Path(path).write_text(qmd_str, encoding='utf-8')
-def write_nb_or_qmd(nb, path):
+def write_nb_or_qmd(nb: AttrDict, path:str):
if Path(path).suffix == '.qmd': write_qmd(nb, path)
else: write_nb(nb, path)
diff --git a/nbs/api/09_frontmatter.ipynb b/nbs/api/09_frontmatter.ipynb
index cddf629a2..139ab6156 100644
--- a/nbs/api/09_frontmatter.ipynb
+++ b/nbs/api/09_frontmatter.ipynb
@@ -78,7 +78,7 @@
"_re_fm_title_desc = re.compile(r'^#\\s+(\\S.*?)(?:\\n|$)(?:\\s*\\n)*(?:>\\s+(\\S.*?)(?:\\n|$)(?:\\s*\\n)*)?', flags=re.MULTILINE)\n",
"_re_fm_kv = re.compile(r'^((?:\\s*-\\s+[a-zA-Z_][a-zA-Z0-9_-]*\\s*:\\s+.*(?:\\n|$)|\\s*\\n)*)', flags=re.MULTILINE)\n",
"\n",
- "def _parse_kv_block(block_text):\n",
+ "def _parse_kv_block(block_text: str):\n",
" \"\"\"Parse a block of key-value pairs from lines starting with '-'\"\"\"\n",
" if not block_text.strip(): return {}\n",
" kv_lines = []\n",
@@ -94,7 +94,7 @@
" return {}\n",
"\n",
"def _md2dict(s:str):\n",
- " \"Convert H1 formatted markdown cell to frontmatter dict\"\n",
+ " \"Convert custom H1 formatted markdown cell to frontmatter dict\"\n",
" if '#' not in s: return {}\n",
" res = {}\n",
" remaining_start = 0\n",
diff --git a/nbs/api/15_qmd.ipynb b/nbs/api/15_qmd.ipynb
index 7cc6b4a02..304c7952c 100644
--- a/nbs/api/15_qmd.ipynb
+++ b/nbs/api/15_qmd.ipynb
@@ -62,22 +62,20 @@
"outputs": [],
"source": [
"#|export\n",
- "def _qmd_to_raw_cell(source_str, cell_type_str, qmd_metadata=None):\n",
+ "def _qmd_to_raw_cell(source:str, cell_type:str, qmd_metadata=None):\n",
" \"\"\"Create a default ipynb json cell\"\"\"\n",
- " cell = {'cell_type': cell_type_str, 'metadata': {}, 'source': source_str}\n",
- " if cell_type_str == 'code':\n",
+ " cell = {'cell_type': cell_type, 'metadata': {}, 'source': source}\n",
+ " if cell_type == 'code':\n",
" cell['execution_count'] = None\n",
" cell['outputs'] = []\n",
" if qmd_metadata: cell['qmd_metadata'] = qmd_metadata\n",
" return cell\n",
"\n",
- "def read_qmd(path): \n",
+ "def read_qmd(path:str): \n",
" \"\"\"Reads a .qmd file as an nb compatible with the rest of execnb and nbdev\"\"\"\n",
" content = Path(path).read_text(encoding='utf-8')\n",
- " # Modified regex to capture the metadata between {python and }\n",
" cell_pat = re.compile(r\"^(`{3,})\\s*\\{python([^\\}]*)\\}\\s*\\n(.*?)^\\1\\s*$\", re.MULTILINE | re.DOTALL)\n",
" \n",
- " # `parts` will be [md_chunk, captured_backticks_1, captured_metadata_1, captured_code_1, md_chunk_2, ...]\n",
" parts = cell_pat.split(content)\n",
" raw_cells = []\n",
" \n",
@@ -87,11 +85,11 @@
" \n",
" # 4 items per match: [md, backticks, metadata?, code]\n",
" for i in range(1, len(parts), 4):\n",
- " if i + 2 < len(parts): # We have backticks, metadata, and code\n",
+ " if i + 2 < len(parts): # backticks, metadata, code\n",
" metadata = parts[i+1] # The captured metadata\n",
" code_source = parts[i+2].strip() # The captured code\n",
" if code_source: raw_cells.append(_qmd_to_raw_cell(code_source, 'code', metadata if metadata else None))\n",
- " if i + 3 < len(parts): # Intermediate markdown\n",
+ " if i + 3 < len(parts): # intermediate markdown\n",
" intermediate_md_source = parts[i+3].strip()\n",
" if intermediate_md_source: raw_cells.append(_qmd_to_raw_cell(intermediate_md_source, 'markdown'))\n",
" \n",
@@ -110,7 +108,7 @@
" \n",
" return dict2nb(notebook_dict)\n",
"\n",
- "def read_nb_or_qmd(path):\n",
+ "def read_nb_or_qmd(path:str):\n",
" if Path(path).suffix == '.qmd': return read_qmd(path)\n",
" return read_nb(path)\n"
]
@@ -200,7 +198,7 @@
"outputs": [],
"source": [
"#|export\n",
- "def _get_fence_ticks(source):\n",
+ "def _get_fence_ticks(source:str):\n",
" \"\"\"Determine the number of backticks needed for fencing that won't conflict with source\"\"\"\n",
" if '`' not in source: return '```' # Default to 3 if no backticks in source\n",
" \n",
@@ -214,7 +212,7 @@
" while num_ticks in used_lengths: num_ticks += 1\n",
" return '`' * num_ticks\n",
"\n",
- "def _nb_to_qmd_str(nb):\n",
+ "def _nb_to_qmd_str(nb: AttrDict):\n",
" \"\"\"Convert a notebook to a string in .qmd format\"\"\"\n",
" def cell_to_qmd(cell):\n",
" source = cell.source.rstrip('\\n')\n",
@@ -227,12 +225,12 @@
" return ''\n",
" return '\\n\\n'.join(filter(None, [cell_to_qmd(cell) for cell in nb.cells]))\n",
"\n",
- "def write_qmd(nb, path):\n",
+ "def write_qmd(nb: AttrDict, path:str):\n",
" \"\"\"Write a notebook back to .qmd format\"\"\"\n",
" qmd_str = _nb_to_qmd_str(nb)\n",
" Path(path).write_text(qmd_str, encoding='utf-8')\n",
" \n",
- "def write_nb_or_qmd(nb, path):\n",
+ "def write_nb_or_qmd(nb: AttrDict, path:str):\n",
" if Path(path).suffix == '.qmd': write_qmd(nb, path)\n",
" else: write_nb(nb, path)\n"
]
From e510a858f2aa4443e3dca8e3feac11022476ce50 Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Tue, 3 Jun 2025 17:10:46 -0400
Subject: [PATCH 30/31] Update tutorial
---
nbs/tutorials/develop_in_plain_text.qmd | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
diff --git a/nbs/tutorials/develop_in_plain_text.qmd b/nbs/tutorials/develop_in_plain_text.qmd
index 3e7fe4e26..3057c7759 100644
--- a/nbs/tutorials/develop_in_plain_text.qmd
+++ b/nbs/tutorials/develop_in_plain_text.qmd
@@ -1,6 +1,8 @@
# Develop in plain text
> Avoid the overhead of developing in .ipynb notebooks while working in plain text.
+- order: 10
+
Using [quarto](https://quarto.org/docs/get-started/hello/vscode.html) and its [VS code extension](https://marketplace.visualstudio.com/items?itemName=quarto.quarto), it is possible to use `.qmd` files instead of `.ipynb` files for a interactive, plain text library development experience using `nbdev`. That `.qmd` files are plain text comes with several advantages:
1. 🤖 `.qmd` **seamlessly integrates with AI copilots** (e.g., Cursor) with no overhead of a special notebook extension
@@ -8,7 +10,7 @@ Using [quarto](https://quarto.org/docs/get-started/hello/vscode.html) and its [V
3. 🪄 `.qmd` **works with your favorite editing style** (e.g., VIM)
4. 🫧 `.qmd` **source files stay clean** during `nbdev`'s transpilation process. You don't need a special `nbdev_clean` step to remove cell metadata and outputs.
-All this while remaining fully interactive thanks to VSCode's [interactive window](https://code.visualstudio.com/docs/python/jupyter-support-py) (see [below](#example))
+All this while remaining fully interactive thanks to VSCode's [interactive window](https://code.visualstudio.com/docs/python/jupyter-support-py).
Starting from version YYY, `nbdev` will automatically support both `.qmd` files and `.ipynb` files for authoring the main library code+docs+tests.
@@ -16,11 +18,6 @@ Starting from version YYY, `nbdev` will automatically support both `.qmd` files
Ensure that all files under your `nbs/` directory have distinct names. E.g., do not have both `00_core.ipynb` and `00_core.qmd`, as both of these will create the intermediate `_proc/00_core.ipynb`
:::
-## Interactively develop in .qmd {#example}
-
-1. Install the [quarto VSCode extension](https://marketplace.visualstudio.com/items?itemName=quarto.quarto)
-2. Open a `.qmd` file in VSCode.
-
## Helpful hotkeys
From 4f23ca8f2d29a4154bbaf3e9645b230d427d70de Mon Sep 17 00:00:00 2001
From: Ben Hoover <24350185+bhoov@users.noreply.github.com>
Date: Tue, 3 Jun 2025 17:27:29 -0400
Subject: [PATCH 31/31] Cleanup
---
.gitignore | 2 --
README.md | 15 --------
nbdev/qmd.py | 13 +------
nbs/api/15_qmd.ipynb | 13 +------
tests/minimal.qmd | 7 ----
tst_index_pandoc.ipynb | 82 ------------------------------------------
6 files changed, 2 insertions(+), 130 deletions(-)
delete mode 100644 tests/minimal.qmd
delete mode 100644 tst_index_pandoc.ipynb
diff --git a/.gitignore b/.gitignore
index 9c6eae719..555cb221d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -149,5 +149,3 @@ checklink/cookies.txt
# .gitconfig is now autogenerated
.gitconfig
-
-nbdev_bak/
\ No newline at end of file
diff --git a/README.md b/README.md
index 870eaf572..ba5728950 100644
--- a/README.md
+++ b/README.md
@@ -79,12 +79,10 @@ available commands:
!nbdev_help
```
- nb_export Export a single nbdev notebook to a python script.
nbdev_bump_version Increment version in settings.ini by one
nbdev_changelog Create a CHANGELOG.md file from closed and labeled GitHub issues
nbdev_clean Clean all notebooks in `fname` to avoid merge conflicts
nbdev_conda Create a `meta.yaml` file ready to be built into a package, and optionally build and upload it
- nbdev_contributing Create CONTRIBUTING.md from contributing_nb (defaults to 'contributing.ipynb' if present). Skips if the file doesn't exist.
nbdev_create_config Create a config file.
nbdev_docs Create Quarto docs and README.md
nbdev_export Export notebooks in `path` to Python modules
@@ -94,18 +92,6 @@ available commands:
nbdev_install Install Quarto and the current library
nbdev_install_hooks Install Jupyter and git hooks to automatically clean, trust, and fix merge conflicts in notebooks
nbdev_install_quarto Install latest Quarto on macOS or Linux, prints instructions for Windows
- nbdev_ipynb_to_qmd
- Converts .ipynb files from source_folder to .qmd files in dest_folder.
- Other files are copied directly. Replicates directory structure.
-
- Warning, you will need to manually check the generated .qmd files for:
- 1. **Code blocks that contain 3 backticks**.
- All code blocks are exported using 3 backticks. Any codeblocks with python strings containing 3 consecutive backticks will break.
- Manually change the code fences to use 4+ backticks if you need to include 3 consecutive backticks in a code block.
-
- 2. **Frontmatter encoded using lists of KV pairs**.
- This is not supported in .qmd files. You will need to manually add the frontmatter to the .qmd files in the standard frontmatter format.
-
nbdev_merge Git merge driver for notebooks
nbdev_migrate Convert all markdown and notebook files in `path` from v1 to v2
nbdev_new Create an nbdev project.
@@ -123,7 +109,6 @@ available commands:
nbdev_trust Trust notebooks matching `fname`
nbdev_update Propagate change in modules matching `fname` to notebooks that created them
nbdev_update_license Allows you to update the license of your project.
- watch_export Use `nb_export` on ipynb files in `nbs` directory on changes using nbdev config if available
## FAQ
diff --git a/nbdev/qmd.py b/nbdev/qmd.py
index b419828a5..98d5e88a4 100644
--- a/nbdev/qmd.py
+++ b/nbdev/qmd.py
@@ -193,18 +193,7 @@ def ipynb_to_qmd(
source_folder: str, # Source folder containing .ipynb files
dest_folder: str # Destination folder for .qmd files and copied items
):
- """
- Converts .ipynb files from source_folder to .qmd files in dest_folder.
- Other files are copied directly. Replicates directory structure.
-
- Warning, you will need to manually check the generated .qmd files for:
- 1. **Code blocks that contain 3 backticks**.
- All code blocks are exported using 3 backticks. Any codeblocks with python strings containing 3 consecutive backticks will break.
- Manually change the code fences to use 4+ backticks if you need to include 3 consecutive backticks in a code block.
-
- 2. **Frontmatter encoded using lists of KV pairs**.
- This is not supported in .qmd files. You will need to manually add the frontmatter to the .qmd files in the standard frontmatter format.
- """
+ "Converts .ipynb files from source_folder to .qmd files in dest_folder. Other files are copied directly."
source_dir = Path(source_folder)
dest_dir = Path(dest_folder)
diff --git a/nbs/api/15_qmd.ipynb b/nbs/api/15_qmd.ipynb
index 304c7952c..986c264ff 100644
--- a/nbs/api/15_qmd.ipynb
+++ b/nbs/api/15_qmd.ipynb
@@ -390,18 +390,7 @@
" source_folder: str, # Source folder containing .ipynb files\n",
" dest_folder: str # Destination folder for .qmd files and copied items\n",
"):\n",
- " \"\"\"\n",
- " Converts .ipynb files from source_folder to .qmd files in dest_folder.\n",
- " Other files are copied directly. Replicates directory structure.\n",
- " \n",
- " Warning, you will need to manually check the generated .qmd files for:\n",
- " 1. **Code blocks that contain 3 backticks**. \n",
- " All code blocks are exported using 3 backticks. Any codeblocks with python strings containing 3 consecutive backticks will break. \n",
- " Manually change the code fences to use 4+ backticks if you need to include 3 consecutive backticks in a code block.\n",
- " \n",
- " 2. **Frontmatter encoded using lists of KV pairs**. \n",
- " This is not supported in .qmd files. You will need to manually add the frontmatter to the .qmd files in the standard frontmatter format.\n",
- " \"\"\"\n",
+ " \"Converts .ipynb files from source_folder to .qmd files in dest_folder. Other files are copied directly.\"\n",
" source_dir = Path(source_folder)\n",
" dest_dir = Path(dest_folder)\n",
"\n",
diff --git a/tests/minimal.qmd b/tests/minimal.qmd
deleted file mode 100644
index 3c46a4551..000000000
--- a/tests/minimal.qmd
+++ /dev/null
@@ -1,7 +0,0 @@
-## A minimal notebook
-
-
-```{python}
-# Do some arithmetic
-1+1
-```
\ No newline at end of file
diff --git a/tst_index_pandoc.ipynb b/tst_index_pandoc.ipynb
deleted file mode 100644
index cbe085536..000000000
--- a/tst_index_pandoc.ipynb
+++ /dev/null
@@ -1,82 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "------------------------------------------------------------------------\n",
- "\n",
- "## title: \"HAMUX QMD\" format: gfm+yaml_metadata_block\n",
- "\n",
- "# HAMUX\n",
- "\n",
- "``` {python}\n",
- "#| echo: false\n",
- "# from hamux_qmd.core import *\n",
- "```\n",
- "\n",
- "> Energy formulation for deep learning\n",
- "\n",
- "This file will become your README and also the index of your\n",
- "documentation.\n",
- "\n",
- "## Developer Guide\n",
- "\n",
- "If you are new to using `nbdev` here are some useful pointers to get you\n",
- "started.\n",
- "\n",
- "### Install hamux_qmd in Development mode\n",
- "\n",
- "``` sh\n",
- "# make sure hamux_qmd package is installed in development mode\n",
- "$ pip install -e .\n",
- "\n",
- "# make changes under nbs/ directory\n",
- "# ...\n",
- "\n",
- "# compile to have changes apply to hamux_qmd\n",
- "$ nbdev_prepare\n",
- "```\n",
- "\n",
- "## Usage\n",
- "\n",
- "### Installation\n",
- "\n",
- "Install latest from the GitHub\n",
- "[repository](https://github.com/bhoov/hamux_qmd):\n",
- "\n",
- "``` sh\n",
- "$ pip install git+https://github.com/bhoov/hamux_qmd.git\n",
- "```\n",
- "\n",
- "or from [conda](https://anaconda.org/bhoov/hamux_qmd)\n",
- "\n",
- "``` sh\n",
- "$ conda install -c bhoov hamux_qmd\n",
- "```\n",
- "\n",
- "or from [pypi](https://pypi.org/project/hamux_qmd/)\n",
- "\n",
- "``` sh\n",
- "$ pip install hamux_qmd\n",
- "```\n",
- "\n",
- "## How to use\n",
- "\n",
- "Fill me in please! Don't forget code examples:\n",
- "\n",
- "``` {python}\n",
- "1+1\n",
- "```\n",
- "\n",
- "``` {python}\n",
- "1+1 \n",
- "```"
- ],
- "id": "f0de16ee-b9c9-4177-899b-0d78ff3d438e"
- }
- ],
- "nbformat": 4,
- "nbformat_minor": 5,
- "metadata": {}
-}