def blocklist_scan(blocks: list[Block], default_title: str = "Title") -> list[Block]:
    """General check that the markdown is suitable for work,
    returning a list of blocks with a header block first.

    Args:
        blocks: the list of blocks to process.
        default_title: the default title to use when no title is found
            or when the title is "Title".

    Returns:
        the processed list of blocks. If the input contains only
        ErrorBlocks, returns the list as-is without adding a header,
        as ErrorBlocks signal that the block list is not valid.

    Examples:
        ```
        >>> from lmm.markdown.parse_markdown import parse_markdown_text
        >>> blocks = parse_markdown_text("# My Document\\n\\nSome text")
        >>> result = blocklist_scan(blocks)
        >>> isinstance(result[0], HeaderBlock)
        True
        >>> result[0].content['title']
        'My Document'
        ```
    """

    if not blocks:  # Empty list
        return [HeaderBlock.from_default()]

    # Validate first block and ensure first block is header,
    # creating one if necessary
    match blocks[0]:
        case HeaderBlock() | MetadataBlock() as bl:
            if (
                'title' not in bl.content
                or bl.content['title'] == "Title"
            ):
                bl.content['title'] = default_title
                if not bl.comment:
                    bl.comment = "**Default title added**"
            # replace first with header
            blocks[0] = HeaderBlock._from_metadata_block(bl)
        case HeadingBlock() as bl:
            metadata: MetadataDict = {'title': bl.content}
            blocks.insert(
                0,
                HeaderBlock(
                    content=metadata,
                    comment="**Default header added**",
                ),
            )
        case TextBlock():
            metadata: MetadataDict = {'title': default_title}
            blocks.insert(
                0,
                HeaderBlock(
                    content=metadata,
                    comment="**Default header added**",
                ),
            )
        case ErrorBlock():
            pass

    return blocks

@validate_call(config={'arbitrary_types_allowed': True})
def markdown_scan(
    sourcefile: str | Path,
    save: bool | str | Path = True,
    *,
    max_size_mb: float = 50.0,
    warn_size_mb: float = 10.0,
    logger: LoggerBase = logger,
) -> list[Block]:
    """General check that the markdown is suitable for work,
    returning a list of blocks with a header block first. When
    a title is missing, uses the filename stem as the default title.

    Args:
        sourcefile: the file to load the markdown from
        save: if False, does not save; if True, saves back to
            original markdown file; if a filename, saves to
            file.
        max_size_mb: the max size, in MB, of the file to load
        warn_size_mb: the size of the input file that results in
            a warning
        logger: a logger object (defaults to console logging)

    Returns:
        the processed list of blocks.

    Note:
        if an error occurs and the blocklist becomes empty,
        it does not alter the source file.

    Examples:
        ```python
        # Scan a markdown file and save changes. Timestamp added
        blocks = markdown_scan("document.md", save=True)

        # Scan without saving
        blocks = markdown_scan("document.md", save=False)

        # Scan and save to different file, timestamp added
        blocks = markdown_scan("source.md", save="output.md")
        ```
    """

    # Source validation
    source: Path | None = iou.validate_file(sourcefile, logger)
    if not source:
        return []
    # For type-checking
    source = Path(source)

    # load_blocks is guaranteed to return an empty list or a list
    # of blocks.
    blocks: list[Block] = mkd.load_blocks(
        source,
        max_size_mb=max_size_mb,
        warn_size_mb=warn_size_mb,
        logger=logger,
    )
    if not blocks:  # Empty list check
        logger.warning(f"No blocks found in file: {source}")
        return []
    if mkd.blocklist_haserrors(blocks):
        logger.warning(f"Errors found while scanning {source}")

    # Use blocklist_scan with filename stem as default title
    # This ensures missing titles are replaced with the filename
    blocks = blocklist_scan(blocks, default_title=source.stem)
    if not blocks:
        return []

    # Save and return
    match save:
        case False:
            pass
        case True:
            save_scan(source, blocks, logger=logger)
        case str() | Path():
            save_markdown(save, blocks, logger=logger)
        case _:  # ignore
            pass

    return blocks

@validate_call(config={'arbitrary_types_allowed': True})
def save_scan(
    destfile: str | Path,
    blocks: list[Block],
    *,
    verify_unchanged: bool = True,
    logger: LoggerBase = logger,
) -> bool:
    """
    Save blocks to markdown file with optional timestamp verification.

    This function provides a safe save mechanism that can verify the file
    hasn't been modified since it was loaded, preventing accidental overwrites
    of concurrent changes. A timestamp is stored in the header metadata block
    using the key '~last_modified'.

    Args:
        destfile: Destination file path (string or Path object)
        blocks: List of Block objects to save (must have HeaderBlock first)
        verify_unchanged: If True, check timestamp to verify file hasn't 
            changed since load. Defaults to True for safety.
        logger: Logger object for error reporting

    Returns:
        True if saved successfully, False otherwise

    Examples:
        ```
        >>> # Basic save to new file
        >>> from lmm.scan.scan import markdown_scan, save_scan
        >>> blocks = markdown_scan("test.md", save=False)
        >>> save_scan("output.md", blocks)
        True

        >>> # Load, modify, and save with verification
        >>> blocks = markdown_scan("test.md", save=False)
        >>> blocks[0].content['author'] = 'New Author'
        >>> save_scan("test.md", blocks, verify_unchanged=True)
        True

        >>> # Force save without verification
        >>> save_scan("test.md", blocks, verify_unchanged=False)
        True
        ```

    Note:
        - The timestamp is stored in blocks[0].content['~last_modified']
        - If verify_unchanged=True and timestamps don't match, returns False
        - Missing timestamps are handled gracefully (first save or legacy file)
        - Errors are logged through the logger object
    """
    # Validate inputs
    if not blocks:
        logger.error("Cannot save empty block list")
        return False

    if not isinstance(blocks[0], HeaderBlock):
        logger.error("First block must be a HeaderBlock")
        return False

    # Convert to Path
    dest_path = Path(destfile)

    # Verify timestamp if requested and file exists
    if verify_unchanged and dest_path.exists():
        try:
            # Load existing file to check timestamp
            existing_blocks = markdown_scan(dest_path, save=False, logger=logger)

            if not existing_blocks:
                logger.warning(
                    f"Could not load existing file {dest_path} for "
                    "timestamp verification, proceeding anyway"
                )
            else:
                # Get timestamps
                existing_timestamp: str = existing_blocks[0].content.get(LAST_MODIFIED_KEY) # type: ignore
                current_timestamp: str = blocks[0].content.get(LAST_MODIFIED_KEY) # type: ignore

                # Compare timestamps
                if existing_timestamp and current_timestamp:
                    if existing_timestamp != current_timestamp:
                        logger.warning(
                            f"File {dest_path} has been modified since load. "
                            f"Expected timestamp: {current_timestamp}, "
                            f"found: {existing_timestamp}. Save aborted."
                        )
                        return False
                elif existing_timestamp and not current_timestamp:
                    logger.info(
                        f"Blocks to save have no timestamp, but file {dest_path} "
                        "does. This may indicate the blocks were not loaded via "
                        "save_scan. Proceeding with save."
                    )
                # If neither has timestamp or only current has one, proceed

        except Exception as e:
            logger.error(f"Error during timestamp verification: {e}")
            return False

    # Update timestamp with current time
    blocks[0].content[LAST_MODIFIED_KEY] = datetime.now().isoformat()

    # Save using save_markdown
    success = save_markdown(dest_path, blocks, logger)

    if success:
        logger.info(f"Successfully saved {dest_path}")

    return success

def scan(
    sourcefile: str | Path,
    save: bool | str | Path = True,
    *,
    max_size_mb: float = 50.0,
    warn_size_mb: float = 10.0,
    logger: LoggerBase = logger,
) -> None:
    """General check that the markdown is suitable for work.
    This is a wrapper around markdown_scan that catches exceptions and 
    logs them, suitable for command-line interface use.

    Args:
        sourcefile: the file to load the markdown from
        save: if False, does not save; if True, saves back to
            original markdown file; if a filename, saves to
            file.
        max_size_mb: the max size, in MB, of the file to load
        warn_size_mb: the size of the input file that results in
            a warning
        logger: a logger object (defaults to console logging)

    Returns:
        None. Errors are logged instead of raised.
    """

    try:
        markdown_scan(
            sourcefile,
            save,
            max_size_mb=max_size_mb,
            warn_size_mb=warn_size_mb,
            logger=logger,
        )
    except (OSError, ValueError, RuntimeError) as e:
        logger.error(f"Error scanning {sourcefile}: {e}")

def blocklist_clear_messages(
    blocks: list[Block], keys: list[str] | None = None
) -> list[Block]:
    """Remove language model interactions from metadata. If specific
    keys are specified, only remove those keys.

    Args:
        blocks: the block list to handle
        keys (opts): specify the keys to remove. Otherwise, will
            remove the keys used in message exchanges.
    """

    if keys is not None:
        return clear_metadata_properties(blocks, keys)

    blocklist: list[Block] = []
    for b in blocks:
        if isinstance(b, MetadataBlock):
            newb: MetadataBlock = b.deep_copy()
            kks: dict_keys[str, MetadataValue] = newb.content.keys()
            if QUERY_KEY in kks:
                newb.content.pop(QUERY_KEY)
            if MESSAGE_KEY in kks:
                newb.content.pop(MESSAGE_KEY)
            if EDIT_KEY in kks:
                newb.content.pop(EDIT_KEY)
            if CHAT_KEY in kks:
                newb.content.pop(CHAT_KEY)

            if len(newb.content) > 0 or bool(newb.private_):
                blocklist.append(newb)
        else:
            blocklist.append(b)

    return blocklist

def blocklist_messages(
    blocks: list[Block], logger: LoggerBase = logger
) -> list[Block]:
    """
    Carries out the interaction with the language model,
    returning a list of blocks with a header block first.

    Args:
        blocks: markdown blocks to process

    Returns:
        the processed list of blocks.
    """

    if not blocks:
        return []

    blocks = blocklist_scan(blocks)
    if blocklist_haserrors(blocks):
        logger.warning("Problems in markdown, fix before continuing")
        return blocks

    root: MarkdownTree = blocks_to_tree(
        blocklist_copy(blocks), logger
    )
    if not root:
        return []

    processed_root: MarkdownNode = _process_chain(root, logger)
    return tree_to_blocks(processed_root)

@validate_call(config={'arbitrary_types_allowed': True})
def markdown_clear_messages(
    sourcefile: str | Path,
    keys: list[str] | None = None,
    save: bool | str | Path = True,
    logger: LoggerBase = logger,
) -> list[Block]:
    """
    Removes the messages from a markdown. If keys is specified,
    removes the metadata properties specified by keys.

    Args:
        sourcefile: the file to load the markdown from
        keys (optional): the keys of messages or any property to
            remove
        save: if False, does not save; if True, saves back to
            original markdown file; if a filename, saves to
            file.

    Note:
        if an error occurs and the blocklist becomes empty,
        it does not alter the source file.
    """

    SAVE_FILE = False
    blocks: list[Block] = markdown_scan(
        sourcefile, SAVE_FILE, logger=logger
    )
    if not blocks:
        return []

    if blocklist_haserrors(blocks):
        save_markdown(sourcefile, blocks, logger)
        logger.warning("Problems in markdown, fix before continuing")
        return []

    blocks = blocklist_clear_messages(blocks, keys)

    match save:
        case False:
            pass
        case True:
            save_markdown(sourcefile, blocks, logger)
        case str() | Path():
            save_markdown(save, blocks, logger)
        case _:  # ignore
            pass

    return blocks

@validate_call(config={'arbitrary_types_allowed': True})
def markdown_messages(
    sourcefile: str | Path,
    save: bool | str | Path = True,
    *,
    max_size_mb: float = 50.0,
    warn_size_mb: float = 10.0,
    logger: LoggerBase = logger,
) -> list[Block]:
    """
    Carries out the interaction with the language model,
    returning a list of blocks with a header block first.

    Args:
        sourcefile: the file to load the markdown from
        save: if False, does not save; if True, saves back to
            original markdown file; if a filename, saves to
            file.

    Note:
        if an error occurs and the blocklist becomes empty,
        it does not alter the source file.
    """

    SAVE_FILE = False
    blocks: list[Block] = markdown_scan(
        sourcefile,
        SAVE_FILE,
        max_size_mb=max_size_mb,
        warn_size_mb=warn_size_mb,
        logger=logger,
    )
    if not blocks:
        return []
    if blocklist_haserrors(blocks):
        save_markdown(sourcefile, blocks, logger)
        logger.warning("Problems in markdown, fix before continuing")
        return []

    root: MarkdownTree = blocks_to_tree(
        blocklist_copy(blocks), logger
    )
    if not root:
        return []

    processed_root: MarkdownNode = _process_chain(root, logger)
    blocks = tree_to_blocks(processed_root)
    if not blocks:
        return []

    match save:
        case False:
            pass
        case True:
            save_markdown(sourcefile, blocks, logger)
        case str() | Path():
            save_markdown(save, blocks, logger)
        case _:  # ignore
            pass

    return blocks

def scan_clear_messages(
    sourcefile: str | Path,
    keys: list[str] | None = None,
    save: bool | str | Path = True,
    logger: LoggerBase = logger,
) -> None:
    """
    Removes the messages from a markdown. If keys is specified,
    removes the metadata properties specified by keys.

    Args:
        sourcefile: the file to load the markdown from
        keys (optional): the keys of messages or any property to
            remove
        save: if False, does not save; if True, saves back to
            original markdown file; if a filename, saves to
            file.

    Note:
        stub of markdown_clear_messages for interface build
    """

    try:
        markdown_clear_messages(sourcefile, keys, save, logger)
    except Exception as e:
        logger.error(str(e))

def scan_messages(
    sourcefile: str | Path,
    save: bool | str | Path = True,
    *,
    max_size_mb: float = 50.0,
    warn_size_mb: float = 10.0,
    logger: LoggerBase = logger,
) -> None:
    """
    Carries out the interaction with the language model,
    returning a list of blocks with a header block first.

    Args:
        sourcefile: the file to load the markdown from
        save: if False, does not save; if True, saves back to
            original markdown file; if a filename, saves to
            file.

    Note:
        stub of markdown_messages for interface build
    """

    try:
        markdown_messages(
            sourcefile,
            save,
            max_size_mb=max_size_mb,
            warn_size_mb=warn_size_mb,
            logger=logger,
        )
    except Exception as e:
        logger.error(str(e))

def add_id_to_nodes(
    root_node: MarkdownNode,
    textid: bool,
    headingid: bool,
    base_hash: str | None,
    logger: LoggerBase,
    *,
    filt_func: Callable[[MarkdownNode], bool] = lambda _: True,
) -> None:
    """Add unique identifiers to text and heading blocks in a markdown
    tree. These identifiers may be used when ingesting the document,
    to create the id's used by the vector database, such that new
    versions of the same blocks are overwritten in the database.

    This function traverses the markdown tree and assigns unique
    identifiers to TextBlock and HeadingBlock nodes. The identifiers
    are constructed using:
    1. A base hash derived from the document's title or a provided
        base_hash
    2. A sequential counter for text and heading blocks

    The function adds two types of metadata identifiers:
    - 'textid': Unique identifier for TextBlock nodes
    - 'headingid': Unique identifier for HeadingBlock nodes

    Args:
        root_node (MarkdownNode): The root node of the markdown tree
            to process
        textid: add id to text nodes (boolean)
        headingid: add id to heading nodes (boolean)
        base_hash (str, optional): A base hash to use for identifier
            generation. If not provided, a hash is generated from the
            root node's content (and will differ from that content).
        logger: a logger object.
        filt_func: a predicate to filter the nodes where the id should
            be added.

    Identifier Format:
    - For text blocks: "{base_hash}.{sequential_number}"
      Example: "abc123.1", "abc123.2"
    - For heading blocks: "{base_hash}.h{sequential_number}"
      Example: "abc123.h1", "abc123.h2"

    Note:
        - Identifiers are _always_ added irrespective of whether they
            already exist in the node's metadata.
        - The base_hash function produces a hash derived from the docid,
            ensuring uniqueness across different documents while maintaining
            deterministic IDs for the same document content.
    """

    textid = bool(textid)
    headingid = bool(headingid)
    if not (textid or headingid):
        return

    if not base_hash:
        from lmm.utils.hash import base_hash as hash_func

        title: str = root_node.get_content()
        docid: str = root_node.get_metadata_string_for_key(  # type: ignore
            DOCID_KEY, title  # title is default if DOCID missing
        )
        # don't just use docid, make it same length and intuitively
        # not something to tamper with.
        base_hash = hash_func(docid)

    counter: dict[str, int] = {'text': 0, 'heading': 0}
    textkey = TEXTID_KEY
    headingkey = HEADINGID_KEY

    def _add_id(node: MarkdownNode) -> None:
        if not filt_func(node):
            return
        match node.block:
            case TextBlock() if textid:
                counter['text'] += 1
                node.metadata[textkey] = (
                    f"{base_hash}.{counter['text']}"
                )
            case HeadingBlock() if headingid:
                counter['heading'] += 1
                node.metadata[headingkey] = (
                    f"{base_hash}.h{counter['heading']}"
                )
            case _:
                pass

    pre_order_traversal(root_node, _add_id)

def add_questions(
    root: MarkdownNode,
    opts: ScanOpts,
    logger: LoggerBase,
    *,
    filt_func: Callable[[MarkdownNode], bool] = lambda _: True,
) -> None:
    """Add questions answered by text using a language model. Will
    not add questions to the header node, but to all heading nodes
    in the document.

    Args:
        root: a markdown node to start the traversal
        opts: options defining thresholds for computing questions
        logger: a logger object
        filt_func: a predicated to fiter the heading nodes to add
            questions to.

    Returns: None.
    """

    def llm_questions(text: str) -> str:
        if len(text.split()) < opts.questions_threshold:
            return ""
        response: str = ""
        try:
            kernel: RunnableType = create_runnable(
                "question_generator", opts.language_model_settings
            )
            response = kernel.invoke({'text': text})
        except ConnectionError:
            logger.error(
                "Could not connect to language models.\n"
                + "Check the internet connection."
            )
        except Exception as e:
            logger.error(
                "Error in using the language model to create "
                f"questions: {e}"
            )

        # replace numbers
        pattern = r"\s*\d+[.)]\s*"
        response = re.sub(pattern, "~_~", response)
        return " - ".join(response.split("~_~"))

    # do not call questions at header node.
    if root.is_header_node():
        for node in root.children:
            post_order_hashed_aggregation(
                node,
                llm_questions,
                QUESTIONS_KEY,
                filter_func=filt_func,
            )
    else:
        post_order_hashed_aggregation(
            root, llm_questions, QUESTIONS_KEY, filter_func=filt_func
        )

def add_summaries(
    root: MarkdownNode,
    opts: ScanOpts,
    logger: LoggerBase,
    *,
    filt_func: Callable[[MarkdownNode], bool] = lambda _: True,
) -> None:
    """Add summaries of text to metadata of headings using a
    language model.

    Args:
        root: a markdown node to start the traversal
        opts: options defining thresholds for computing summaries
        logger: a logger object
        filt_func: a predicate function to filter the heading
            noted where a summary will be added.
    """

    def llm_add_summary(text: str) -> str:
        if len(text.split()) < opts.summary_threshold:
            return ""
        response: str = ""
        try:
            kernel: RunnableType = create_runnable(
                runnable_name="summarizer",
                user_settings=opts.language_model_settings,
            )
            response = kernel.invoke({'text': text})
        except ConnectionError:
            logger.error(
                "Could not connect to language models.\n"
                + "Check the internet connection."
            )
        except Exception as e:
            logger.error(
                "Error in using the language model to create "
                f"summaries: {e}"
            )

        return response

    post_order_hashed_aggregation(
        root, llm_add_summary, SUMMARY_KEY, filter_func=filt_func
    )

def add_titles_to_headings(
    root: MarkdownNode,
    logger: LoggerBase,
    *,
    key: str = TITLES_KEY,
    filt_func: Callable[[MarkdownNode], bool] = lambda _: True,
) -> None:
    """Recursively add titles to heading blocks in a markdown tree.

    This function maps nodes a markdown tree in a pre-order manner,
    collecting and concatenating the content of ancestor headings
    for each heading block. It adds a metadata field to HeadingBlock
    nodes, which represents the full hierarchical path of headings
    leading to that specific heading.

    Args:
        root: The root node of the markdown tree to process, or
            any other parent node
        logger: a logger object
        key: the key under which the titles are added
        filt_func: A predicate the nodes must satisfy for titles to
            be added

    Note:
        - Only non-empty heading contents are included in the titles
        - The titles are added to the key field in the node's metadata
    """

    def map_func(node: MarkdownNode) -> None:
        # recursively add content of headings to key in metadata
        if isinstance(node, HeadingNode):
            if not filt_func(node):
                return
            if node.parent:
                titles: str = str(
                    node.parent.get_metadata_for_key(key, "")
                )
            else:
                titles = ""
            title: str = node.get_content()
            node.set_metadata_for_key(
                key,
                titles
                + (" - " if titles else "")
                + (title if title else ""),
            )
        return

    pre_order_traversal(root, map_func)

def blocklist_rag(
    blocks: list[Block],
    opts: ScanOpts = ScanOpts(),
    logger: LoggerBase = _logger,
) -> list[Block]:
    """
    Prepares the blocklist structure for RAG (Retrieval Augmented
    Generation) by enhancing it with metadata.

    Args:
        blocks: a markdown block list
        opts: a ScanOpts object
        logger: a logger object (defaults to console logging)

    Returns:
        list[Block]: List of enhanced markdown blocks, or empty list
            if processing fails

    Note:
        The function adds several metadata fields to blocks:

        - docid: Unique document identifier
        - titles: Hierarchical heading path
        - textid/headingid: Unique block identifiers
        - questions: Potential questions answered by the text
        - summary: Content summaries for heading blocks

        The function will return an empty list if the input block
        list contains error blocks. It will add a default header
        if the header is missing, and a docid field to the header
        if this is missing.

        Reentrant Processing:
            Running blocklist_rag multiple times on the same blocks is
            idempotent if the content hasn't changed. IDs are always
            regenerated to ensure integrity, but summaries/questions are
            only recomputed if the text content has changed (detected via
            hash comparison). This ensures external systems never need to
            reference internal IDs, as the parsed markdown is only used
            to export IDs, never to look them up.

    Example of use:
        ```python
        opts = ScanOpts(titles = True) # add titles
        blocks = blocklist_rag(blocks, opts)

        # override language model from config.toml
        opts = ScanOpts(
            questions = True,               # add questions
            language_model_settings = LanguageModelSettings(
                model = "OpenAI/gpt-4o"
            )
        )
        blocks = blocklist_rag(blocks, opts)
        ```
    """

    # Validation
    build_titles = bool(opts.titles)
    build_questions = bool(opts.questions)
    build_summaries = bool(opts.summaries)
    build_textids = bool(opts.textid)
    build_headingids = bool(opts.headingid)
    build_textUUID = bool(opts.textUUID)
    build_headingUUID = bool(opts.headingUUID)
    if build_textUUID and (not build_textids):
        logger.info("bloclist_rag: text id's built to form UUID")
        build_textids = True
    if build_headingUUID and (not build_headingids):
        logger.info("blocklist_rag: heading id's built to form UUID")
        build_headingids = True

    if not (
        build_titles
        or build_questions
        or build_summaries
        or build_textids
        or build_headingids
        or build_textUUID
        or build_headingUUID
    ):
        logger.info("No RAG changes specified for document.")
        return blocks

    # Validate for lm markdown
    blocks = blocklist_scan(blocks)

    # Further document validation
    if not blocks:
        raise RuntimeError(
            "Unreachable code reached: scan function "
            + "should not return an empty list"
        )
    if len(blocks) == 1 and isinstance(blocks[0], ErrorBlock):
        logger.error("Load failed:\n" + str(blocks[0].get_content()))
        return []
    if blocklist_haserrors(blocks):
        # convert markdown errors into logger errors
        for b in blocklist_errors(blocks):
            logger.error(b.get_content())
        logger.error("Errors in markdown. Fix before continuing.")
        return []

    # Preproc text blocks prior to annotations
    blocks = blocklist_copy(blocks)

    # this removes metadata properties that are not relevant
    # to RAG, such as chats
    if opts.remove_messages:
        blocks = blocklist_clear_messages(blocks)

    # Process directives
    root: MarkdownTree = blocks_to_tree(blocks, logger)
    if not root:
        return []
    logger.info("Processing " + root.get_content())

    # add titles for internal use
    add_titles_to_headings(root, logger, key=TITLES_TEMP_KEY)

    # add docid. This should identify the document uniquely.
    # It may be provided by the human user too.
    if DOCID_KEY not in root.metadata:
        # generate a random string to form doc id
        root.metadata[DOCID_KEY] = generate_random_string()

    # Add titles to headings
    if build_titles:
        logger.info("Adding titles to heading metadata.")
        add_titles_to_headings(
            root, logger, key=TITLES_KEY, filt_func=_filt_func
        )

    # Add an id to all heading and text blocks
    add_id_to_nodes(
        root,
        build_textids,
        build_headingids,
        root.get_metadata_string_for_key(DOCID_KEY),
        logger,
        filt_func=_filt_func,
    )

    # Add UUID to text and heading nodes
    def add_headingUUID_func(node: MarkdownNode) -> None:
        if not _filt_func(node):
            return
        if node.has_metadata_key(UUID_KEY):
            return
        if isinstance(node, HeadingNode):
            uuid_base: str | None = node.get_metadata_string_for_key(
                HEADINGID_KEY
            )
            if uuid_base is not None:
                node.set_metadata_for_key(
                    UUID_KEY,
                    generate_uuid(uuid_base),
                )
            else:
                # should not happen given we have generated TXTID's
                logger.warning("Could not set uuid for object")

    if build_headingUUID:
        logger.info("Adding UUIDs to headings.")
        pre_order_traversal(root, add_headingUUID_func)

    def add_textUUID_func(node: MarkdownNode) -> None:
        if not _filt_func(node):
            return
        if node.has_metadata_key(UUID_KEY):
            return
        if isinstance(node, TextNode):
            uuid_base: str | None = node.get_metadata_string_for_key(
                TEXTID_KEY
            )
            if uuid_base is not None:
                node.set_metadata_for_key(
                    UUID_KEY,
                    generate_uuid(uuid_base),
                )
            else:
                # should not happen given we have generated TXTID's
                logger.warning("Could not set uuid for object")

    if build_textUUID:
        logger.info("Adding UUIDs to text blocks.")
        pre_order_traversal(root, add_textUUID_func)

    # Add source
    def add_source_func(node: MarkdownNode) -> None:
        if not _filt_func(node):
            return
        if isinstance(node, HeadingNode):
            if node.is_header_node():
                return
            source: str | None = node.fetch_metadata_string_for_key(
                DOCID_KEY
            )
            if source:
                node.metadata[SOURCE_KEY] = source

    pre_order_traversal(root, add_source_func)

    # Add a summary to heading nodes that is recomputed after changes
    if build_summaries:
        logger.info("Adding summaries about text.")
        add_summaries(root, opts, filt_func=_filt_func, logger=logger)

    # Add questions that the text answers, recomputed if text changes
    # (will use summaries if existing)
    if build_questions:
        logger.info("Adding questions about text.")
        add_questions(root, opts, filt_func=_filt_func, logger=logger)

    # check meta-data without text
    def _warn_empty_text(node: MarkdownNode) -> None:
        if node.is_header_node():
            pass
        elif isinstance(node, HeadingNode):
            if node.metadata:
                if len(node.get_text_children()) == 0:
                    node.metadata[WARNING_KEY] = (
                        "**Add text under this "
                        + "heading to avoid removal of "
                        + "metadata when ingesting**"
                    )
                elif WARNING_KEY in node.metadata:
                    node.metadata.pop(WARNING_KEY, "")
                else:
                    pass
        elif isinstance(node, TextNode):
            if node.metadata:
                if not node.get_content():
                    node.metadata[WARNING_KEY] = (
                        "**Add text under this "
                        + "metadata to avoid removal of "
                        + "metadata when ingesting**"
                    )
                elif WARNING_KEY in node.metadata:
                    node.metadata.pop(WARNING_KEY, "")
                else:
                    pass
        else:
            pass

    post_order_traversal(root, _warn_empty_text)

    # Re-create blocklist
    blocks = tree_to_blocks(root)

    # remove internal titles
    blocks = clear_metadata_properties(blocks, [TITLES_TEMP_KEY])

    return blocks

def get_changed_titles(
    blocks: list[Block], logger: LoggerBase
) -> list[str]:
    """List the titles of all changed text. This is the
    headings that would be updated in a scan operation.

    Args:
        blocks: the block list to evaluate
        logger: a logger object

    Returns:
        a list of strings containing the headings with
        changed content
    """
    from ..markdown.treeutils import (
        get_nodes_with_metadata,
        get_headingnodes,
    )
    from .scan_keys import TXTHASH_KEY
    from ..utils.logging import LoglistLogger

    internal_logger = LoglistLogger()

    blocklist: list[Block] = blocklist_copy(blocks)
    root: HeadingNode | None = blocks_to_tree(blocklist, logger)
    if root is None:
        return []

    TITLES_KEY = "~__TITLES__"
    OUTPUT_KEY = "~__OUTPUT__"

    # check there are any hashes
    nodes: list[HeadingNode] = get_nodes_with_metadata(
        root, TXTHASH_KEY, HeadingNode
    )
    if not nodes:
        logger.info(
            "No hashes in document (the document has "
            "not been scanned yet)."
        )
        return []

    # add titles to report
    add_titles_to_headings(root, internal_logger, key=TITLES_KEY)
    if internal_logger.count_logs(level=logging.ERROR):
        for log in internal_logger.get_logs(logging.ERROR):
            logger.error(log)
        return []

    # execute hashing. Create output first as otherwise will
    # be reformed irrespective of hash.
    def _add_metadata_func(n: MarkdownNode) -> None:
        n.metadata[OUTPUT_KEY] = "fixed"

    post_order_traversal(root, _add_metadata_func)
    post_order_hashed_aggregation(
        root,
        lambda _: "changed",
        OUTPUT_KEY,
        True,
        filter_func=_filt_func,
        logger=internal_logger,
    )
    if internal_logger.count_logs(level=logging.ERROR):
        for log in internal_logger.get_logs(logging.ERROR):
            logger.error(log)
        return []

    # get nodes when hash discrepancy led to recompute
    nodes: list[HeadingNode] = get_headingnodes(
        root,
        True,
        lambda n: n.get_metadata_string_for_key(OUTPUT_KEY, "")
        == "changed"
        and not n.is_header_node(),
    )

    titles: list[str] = []
    for n in nodes:
        title: str | None = n.get_metadata_string_for_key(TITLES_KEY)
        if title:
            titles.append(title)

    return titles

@validate_call(config={'arbitrary_types_allowed': True})
def markdown_rag(
    sourcefile: str | Path,
    opts: ScanOpts = ScanOpts(),
    save: bool | str | Path = True,
    *,
    max_size_mb: float = 50.0,
    warn_size_mb: float = 10.0,
    logger: LoggerBase = _logger,
) -> list[Block]:
    """
    Scans the markdown file and adds information required for the
    ingestion in the vector database.

    opts defines what operations are conducted on the document,
    but if the header of the document contains an opts field,
    the specifications in the header are used.

    Args:
        sourcefile: the file to load the markdown from
        opts: a ScanOpts objects with the following options:
            titles (False)    add hierarchical titles to headings
            questions (False) add questions to headings
            questions_threshold (15) ignored if questions == False
            summaries (False) add summaries to headings
            summary_threshold (50) ignored if summaries == False
            remove_messages (False)
            textid (False)    add textid to text blocks
            headingid (False) add headingid to headings
            textUUID (False)  add UUID to text blocks
            headingUUID (False) add UUID to heading blocks
            pool_threshold (0) pooling of text blocks
        save: if False, does not save; if True, saves back to
            original markdown file; if a filename, saves to
            file. Defaults to True.
        max_size_mb: the max size, in MB, of the file to load
        warn_size_mb: the size of the input file that results in
            a warning
        logger: a logger object. Defaults to console logger.

    Returns:
        a list of blocks, starting with a header block.

    Note: if an error occurs and the blocklist becomes empty,
        it does not alter the source file.
    """

    blocks: list[Block] = markdown_scan(
        sourcefile,
        False,
        max_size_mb=max_size_mb,
        warn_size_mb=warn_size_mb,
        logger=logger,
    )
    if not blocks:
        return []
    if blocklist_haserrors(blocks):
        save_markdown(sourcefile, blocks, logger)
        logger.warning("Problems in markdown, fix before continuing")
        return []

    # Take over options if specified in header. The isinstance check
    # will always be true since markdown_scan provides a default
    # header if it is missing, but we check for pyright's benefit
    if isinstance(blocks[0], HeaderBlock):
        header: HeaderBlock = blocks[0]
        options: dict[str, object] = header.get_key_type(
            OPTIONS_KEY, dict, {}
        )
        if bool(options):
            logger.info("Reading opts specifications from header")
            try:
                # types checked and coerced by the pydantic model
                opts = ScanOpts(**options)  # type: ignore
            except Exception as e:
                logger.error(f"Invalid scan specification:\n{e}")
                return []
    else:
        raise RuntimeError(
            "Unreachable code reached: header block missing"
        )

    blocks = blocklist_rag(blocks, opts, logger)
    if not blocks:
        return []

    match save:
        case False:
            pass
        case True:
            save_markdown(sourcefile, blocks, logger)
        case str() | Path():
            save_markdown(save, blocks, logger)

    return blocks

def scan_rag(
    sourcefile: str | Path,
    *,
    titles: bool = False,
    questions: bool = False,
    questions_threshold: int = 15,
    summaries: bool = False,
    summary_threshold: int = 50,
    remove_messages: bool = False,
    save: bool | str | Path = True,
    max_size_mb: float = 50.0,
    warn_size_mb: float = 10.0,
    logger: LoggerBase = _logger,
) -> None:
    """Convenience wrapper around markdown_rag with individual parameters.

    This function provides a flattened interface for markdown_rag,
    accepting individual boolean parameters instead of a ScanOpts object.
    Useful for command-line interfaces and simple scripts.

    Args:
        sourcefile: the file to load the markdown from
-           titles (False)    add hierarchical titles to headings
        questions (False) add questions to headings
        questions_threshold (15) ignored if questions == False
        summaries (False) add summaries to headings
        summary_threshold (50) ignored if summaries == False
        remove_messages (False)
        save: if False, does not save; if True, saves back to
            original markdown file; if a filename, saves to
            file. Defaults to True.
        max_size_mb: the max size, in MB, of the file to load
        warn_size_mb: the size of the input file that results in
            a warning
        logger: a logger object. Defaults to console logger.

    Returns: None

    """

    try:
        opts = ScanOpts(
            titles=titles,
            questions=questions,
            questions_threshold=questions_threshold,
            summaries=summaries,
            summary_threshold=summary_threshold,
            remove_messages=remove_messages,
        )
        markdown_rag(
            sourcefile,
            opts,
            save,
            max_size_mb=max_size_mb,
            warn_size_mb=warn_size_mb,
            logger=logger,
        )
    except Exception as e:
        logger.error(str(e))

def blocks_to_splitted_blocks(
    blocks: list[Block], text_splitter: TextSplitter
) -> list[Block]:
    """Transform a blocklist by applying text splitting to the text
    block prior to ingestion. Metadata are inherited from the original
    block.

    Args:
        blocks: a list of markdown blocks
        text_splitter: a langchain text splitter

    Returns:
        a list of markdown blocks
    """

    if isinstance(text_splitter, NullTextSplitter):
        return blocks

    def _split_text_block(bl: TextBlock) -> list[TextBlock]:
        doc: Document = Document(
            page_content=bl.get_content(), metadata={}
        )
        docs = text_splitter.split_documents([doc])
        return [TextBlock(content=d.page_content) for d in docs]

    # split
    newblocks: list[Block] = []
    for b in blocks:
        if isinstance(b, TextBlock):
            splits = _split_text_block(b)
            if newblocks and isinstance(newblocks[-1], MetadataBlock):
                curmeta: MetadataBlock = newblocks[-1].deep_copy()
                newblocks.append(splits[0])
                # do not inherit textid's and UUID's
                curmeta.content.pop(TEXTID_KEY, "")
                curmeta.content.pop(UUID_KEY, "")
                for s in splits[1:]:
                    if curmeta.content:
                        newblocks.append(curmeta.deep_copy())
                    newblocks.append(s)
            else:
                newblocks.extend(splits)
        else:
            newblocks.append(b)
    return newblocks