Dynamically adding new pages does not reflect in on_nav files list

[mihaigalos]

I'm trying to dynamically create index.md files for subfolders which contain no markdown.
These show up in the URL as subpathA/subpathB/subpathC/README.md

If a user manually edits the URL to navigate to say subpathA/subpathB - that's a 404 since the page does not exist.

With this approach, it would contain the listing of the subpath (a list entry with subpathC).

It seems however, that I cannot filter the generated files in on_nav(), since they're not part of the files list, why is that?:

    def on_config(self, config, **kwargs):
        self._setup_logger()
        self.base_url = self._get_base_url(config)
        self.generated_files = []

    def on_files(self, files, config, **kwargs):
        docs_dir = config['docs_dir']
        new_files = []
        for dirpath, _, filenames in os.walk(docs_dir):
            index_path = os.path.join(dirpath, 'index.md')
            if 'index.md' not in filenames:
                self._generate_index_page(docs_dir, dirpath)
                # Add the generated index.md to a look-aside list
                relpath = os.path.relpath(index_path, docs_dir)
                new_files.append(File(relpath, docs_dir, config['site_dir'], config['use_directory_urls']))
                self.generated_files.append(relpath)
        
        for new_file in new_files:
            files.append(new_file)

        # Filter out hidden index files
        filtered_files = [file for file in files if file.src_path not in self.generated_files]

        # Logging for debugging purposes
        for file in filtered_files:
            self.logger.info(f"File in filtered files: {file.src_path}")
        
        return filtered_files

    def _generate_index_page(self, docs_dir, dirpath):
        # Construct the index.md content
        relative_dir = os.path.relpath(dirpath, docs_dir)
        content_lines = [
            "---",
            "hide: true",  # Including metadata to indicate hidden
            "---",
            f"# Index of {relative_dir}",
            ""
        ]
        base_url_part = f"/{self.base_url}/" if self.base_url else "/"

        for item in sorted(os.listdir(dirpath)):
            item_path = os.path.join(dirpath, item)
            if os.path.isdir(item_path):
                content_lines.append(f"- [{item}]({base_url_part}{os.path.join(relative_dir, item)}/index.md)")
            elif item.endswith(".md") and item != "index.md":
                item_name = os.path.splitext(item)[0]
                content_lines.append(f"- [{item_name}]({base_url_part}{os.path.join(relative_dir, item)})")

        content = "
".join(content_lines)
        index_path = os.path.join(dirpath, 'index.md')
        with open(index_path, 'w') as f:
            f.write(content)

        self.logger.info(f"Generated index page: {index_path}")

    def on_nav(self, nav, config, files):
        def filter_nav(items):
            filtered = []
            for item in items:
                if isinstance(item, Section):
                    self.logger.info(f"Processing section: {item.title}")
                    item.children = filter_nav(item.children)
                    if item.children:  
                        filtered.append(item)
                elif isinstance(item, Page):
                    self.logger.info(f"Processing page: {item.url}")
                    if item.file.src_path in self.generated_files and self.generated_files[item.file.src_path].get('hide', False):
                        self.logger.debug(f"Excluding generated file from nav: {item.file.src_path}")
                    else:
                        filtered.append(item)
                else:
                    self.logger.info(f"Processing other: {item.url}")
                    filtered.append(item)
            return filtered

        nav.items = filter_nav(nav.items)
        for nav_item in nav.items:
            self.logger.info(f"Nav item: {nav_item.title}")

        return nav