PathSpec #

// Fs holds the core filesystems used by Hugo.
type Fs struct {
	// Source is Hugo's source file system.
	// Note that this will always be a "plain" Afero filesystem:
	// * afero.OsFs when running in production
	// * afero.MemMapFs for many of the tests.
	Source afero.Fs

	// PublishDir is where Hugo publishes its rendered content.
	// It's mounted inside publishDir (default /public).
	PublishDir afero.Fs

	// WorkingDirReadOnly is a 1.read-only file system
	// restricted to the project working dir.
	WorkingDirReadOnly afero.Fs
}

    p := &Paths{
		Fs:            fs,
		Cfg:           cfg,

		ThemesDir:  cfg.GetString("themesDir"),
		WorkingDir: workingDir,
		AbsPublishDir:   absPublishDir,

		...
	}

	if cfg.IsSet("allModules") {
		p.AllModules = cfg.Get("allModules").(modules.Modules)
	}

// The BasePathFs restricts all operations to a given path within an Fs.
// The given file name to the operations on this Fs will be prepended with
// the base path before calling the base Fs.
// Any file name (after filepath.Clean()) outside this base path will be
// treated as non existing file.
//
// Note that it does not clean the error messages on return, so you may
// reveal the real path on errors.
type BasePathFs struct {
	source Fs
	path   string
}

// NewBaseFileDecorator decorates the given Fs to provide the real filename
// and an Opener func.
func NewBaseFileDecorator(fs afero.Fs) afero.Fs {
	ffs := &baseFileDecoratorFs{Fs: fs}
	decorator := func(fi os.FileInfo, filename string) (os.FileInfo, error) {
		// Store away the original in case it's a symlink.
		meta := NewFileMeta()
		meta.Name = fi.Name()
		...
		opener := func() (afero.File, error) {
			return ffs.open(filename)
		}
		fim := decorateFileInfo(fi, ffs, opener, filename, "", meta)
		return fim, nil
	}
	ffs.decorate = decorator
	return ffs
}

func (fs *baseFileDecoratorFs) Stat(name string) (os.FileInfo, error) {
	fi, err := fs.Fs.Stat(name)
	...
	return fs.decorate(fi, name)
}

func (fs *baseFileDecoratorFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
	...
	if lstater, isLstater := fs.Fs.(afero.Lstater); isLstater {
		fi, ok, err = lstater.LstatIfPossible(name)
	} else {
		fi, err = fs.Fs.Stat(name)
	}
	...
	fi, err = fs.decorate(fi, name)

	return fi, ok, err
}

func newSourceFilesystemsBuilder(p *paths.Paths, b *BaseFs) *sourceFilesystemsBuilder {
	sourceFs := hugofs.NewBaseFileDecorator(p.Fs.Source)
	return &sourceFilesystemsBuilder{p: p, sourceFs: sourceFs, theBigFs: b.theBigFs, result: &SourceFilesystems{}}
}

type sourceFilesystemsBuilder struct {
	p        *paths.Paths
	sourceFs afero.Fs
	result   *SourceFilesystems
	theBigFs *filesystemsCollector
}

// hugo-playground/hugolib/filesystems/basefs.go
// line 454
if b.theBigFs == nil {
    theBigFs, err := b.createMainOverlayFs(b.p)
    ...
    b.theBigFs = theBigFs
}

// hugo-playground/hugolib/filesystems/basefs.go
// line 518
func (b *sourceFilesystemsBuilder) createMainOverlayFs(p *paths.Paths) (*filesystemsCollector, error) {
	collector := &filesystemsCollector{
		...
		overlayDirs:   make(map[string][]hugofs.FileMetaInfo),

		overlayMounts:        overlayfs.New(overlayfs.Options{}),
		overlayMountsContent: overlayfs.New(overlayfs.Options{DirsMerger: hugofs.LanguageDirsMerger}),
		overlayMountsStatic:  overlayfs.New(overlayfs.Options{DirsMerger: hugofs.LanguageDirsMerger}),
		overlayFull:          overlayfs.New(overlayfs.Options{}),
		overlayResources:     overlayfs.New(overlayfs.Options{FirstWritable: true}),
	}

	// Module has same folder structure
	// need to merge, like static files
	mods := p.AllModules
	mounts := make([]mountsDescriptor, len(mods))

	for i := 0; i < len(mods); i++ {
		mod := mods[i]
		dir := mod.Dir()

		isMainProject := mod.Owner() == nil
		mounts[i] = mountsDescriptor{
			Module:        mod,
			dir:           dir,
			isMainProject: isMainProject,
			ordinal:       i,
		}
	}

	err := b.createOverlayFs(collector, mounts)

	return collector, err
}

// hugo-playground/hugolib/filesystems/basefs.go
// line 562
func (b *sourceFilesystemsBuilder) createOverlayFs(
	collector *filesystemsCollector,
	mounts []mountsDescriptor) error {

	for _, md := range mounts {
		var (
			fromTo        []hugofs.RootMapping
			fromToContent []hugofs.RootMapping
			fromToStatic  []hugofs.RootMapping
		)
		...
		for i, mount := range md.Mounts() {
			...
			base, filename := absPathify(mount.Source)
			rm := hugofs.RootMapping{
				From:      mount.Target, // content
				To:        filename,     // mycontent
				ToBasedir: base,
				Module:    md.Module.Path(),
				IsProject: md.isMainProject,
				Meta: ...,
			}
			isContentMount := b.isContentMount(mount)
			...
			if isContentMount {
				fromToContent = append(fromToContent, rm)
			} else if b.isStaticMount(mount) {
				fromToStatic = append(fromToStatic, rm)
			} else {
				fromTo = append(fromTo, rm)
			}
		}
        modBase := collector.sourceProject // source fs, project module
		...
		rmfs, err := hugofs.NewRootMappingFs(modBase, fromTo...)
		...
		rmfsContent, err := hugofs.NewRootMappingFs(modBase, fromToContent...)
		...
		rmfsStatic, err := hugofs.NewRootMappingFs(sourceStatic, fromToStatic...)
		...
		// TODO
		// addDirs
		// overlayMounts
	}
	return nil
}

// hugo-playground/hugofs/rootmapping_fs.go
// line 30
// NewRootMappingFs creates a new RootMappingFs on top of the provided with
// root mappings with some optional metadata about the root.
// Note that From represents a virtual root that maps to the actual filename in To.
func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
	rootMapToReal := radix.New()
	var virtualRoots []RootMapping

	for _, rm := range rms {
		...
		rm.Meta.SourceRoot = rm.To
		rm.Meta.BaseDir = rm.ToBasedir
		rm.Meta.MountRoot = rm.path
		rm.Meta.Module = rm.Module
		rm.Meta.IsProject = rm.IsProject
		
		meta := rm.Meta.Copy()
		...
		rm.fi = NewFileMetaInfo(fi, meta)

		key := filepathSeparator + rm.From // only /layouts and /content
		var mappings []RootMapping
		v, found := rootMapToReal.Get(key)
		...
		mappings = append(mappings, rm)
		rootMapToReal.Insert(key, mappings)

		virtualRoots = append(virtualRoots, rm)
	}
	rootMapToReal.Insert(filepathSeparator, virtualRoots)
	rfs := &RootMappingFs{
		Fs:            fs,
		rootMapToReal: rootMapToReal,
	}
	return rfs, nil
}

// hugo-playground/hugolib/filesystems/basefs.go
// line 562
func (b *sourceFilesystemsBuilder) createOverlayFs(
	collector *filesystemsCollector,
	mounts []mountsDescriptor) error {
        ...
        rmfs, err := hugofs.NewRootMappingFs(modBase, fromTo...)
        ...
        rmfsContent, err := hugofs.NewRootMappingFs(modBase, fromToContent...)
        ...
        rmfsStatic, err := hugofs.NewRootMappingFs(sourceStatic, fromToStatic...)	
	    ...
        // We need to keep the ordered list of directories for watching and
        // some special merge operations (data, i18n).
        collector.addDirs(rmfs)        // add other folders
        collector.addDirs(rmfsContent) // only has /content, why need to go through all components?
        collector.addDirs(rmfsStatic)
        
        getResourcesDir := func() string {
        if md.isMainProject {
        return b.p.AbsResourcesDir
        }
        _, filename := absPathify(files.FolderResources)
        return filename
        }
        
        collector.overlayMounts = collector.overlayMounts.Append(rmfs)
        collector.overlayMountsContent = collector.overlayMountsContent.Append(rmfsContent)
        collector.overlayMountsStatic = collector.overlayMountsStatic.Append(rmfsStatic)
        collector.overlayFull = collector.overlayFull.Append(afero.NewBasePathFs(modBase, md.dir))
        collector.overlayResources = collector.overlayResources.Append(afero.NewBasePathFs(modBase, getResourcesDir()))
	}
	return nil
}

// hugo-playground/hugolib/filesystems/basefs.go
// line 712
func (c *filesystemsCollector) addDirs(rfs *hugofs.RootMappingFs) {
    for _, componentFolder := range files.ComponentFolders {
        c.addDir(rfs, componentFolder)
    }
}

func (c *filesystemsCollector) addDir(rfs *hugofs.RootMappingFs, componentFolder string) {
	dirs, err := rfs.Dirs(componentFolder)

	if err == nil { // event dirs is nil
		// merge all the same component folder from different rfs in the same array
		c.overlayDirs[componentFolder] = append(c.overlayDirs[componentFolder], dirs...)
	}
}

// hugo-playground/hugofs/rootmapping_fs.go
// line 165
func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) {
	base = filepathSeparator + fs.cleanName(base)
	roots := fs.getRootsWithPrefix(base)
	...
	fss := make([]FileMetaInfo, len(roots))
	for i, r := range roots {
		bfs := afero.NewBasePathFs(fs.Fs, r.To)
		...
		fs := bfs
		...
		fi, err := fs.Stat("")
		...
		fss[i] = fi.(FileMetaInfo)
	}
	return fss, nil
}

// hugo-playground/hugofs/rootmapping_fs.go
// line 292
func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping {
	var roots []RootMapping
	fs.rootMapToReal.WalkPrefix(prefix, func(b string, v any) bool {
		roots = append(roots, v.([]RootMapping)...)
		return false
	})

	return roots
}

    ...
    collector.overlayMounts = collector.overlayMounts.Append(rmfs)
    collector.overlayMountsContent = collector.overlayMountsContent.Append(rmfsContent)
    collector.overlayMountsStatic = collector.overlayMountsStatic.Append(rmfsStatic)
    collector.overlayFull = collector.overlayFull.Append(afero.NewBasePathFs(modBase, md.dir))
    collector.overlayResources = collector.overlayResources.Append(afero.NewBasePathFs(modBase, getResourcesDir()))

// Append creates a shallow copy of the filesystem and appends the given filesystems to it.
func (ofs OverlayFs) Append(fss ...afero.Fs) *OverlayFs {
	ofs.fss = append(ofs.fss, fss...)
	return &ofs
}

// hugo-playground/hugolib/filesystems/basefs.go
// line 479
// Data, i18n and content cannot use the overlay fs
dataDirs := b.theBigFs.overlayDirs[files.ComponentFolderData]
dataFs, err := hugofs.NewSliceFs(dataDirs...)
if err != nil {
    return nil, err
}
b.result.Data = b.newSourceFilesystem(files.ComponentFolderData, dataFs, dataDirs)

// hugo-playground/hugolib/filesystems/basefs.go
// line 464
createView := func(componentID string) *SourceFilesystem {
  if b.theBigFs == nil || b.theBigFs.overlayMounts == nil {
      return b.newSourceFilesystem(componentID, hugofs.NoOpFs, nil)
  }
  dirs := b.theBigFs.overlayDirs[componentID]
  return b.newSourceFilesystem(componentID, afero.NewBasePathFs(b.theBigFs.overlayMounts, componentID), dirs)
}
b.result.Layouts = createView(files.ComponentFolderLayouts)

// hugo-playground/hugolib/filesystems/basefs.go
// line 495
contentDirs := b.theBigFs.overlayDirs[files.ComponentFolderContent]
contentBfs := afero.NewBasePathFs(b.theBigFs.overlayMountsContent, files.ComponentFolderContent)
contentFs, err := hugofs.NewLanguageFs(b.p.LanguagesDefaultFirst.AsOrdinalSet(), contentBfs)
if err != nil {
    return nil, fmt.Errorf("create content filesystem: %w", err)
}
b.result.Content = b.newSourceFilesystem(files.ComponentFolderContent, contentFs, contentDirs)

package main

import (
	"bytes"
	"fmt"
	"github.com/sunwei/gobyexample/modules/overlayfs"
	"github.com/sunwei/gobyexample/modules/radixtree"
	"golang.org/x/tools/txtar"
	"io"
	"io/fs"
	"os"
	"path"
	"path/filepath"
)

func main() {
	dir, _ := os.MkdirTemp("", "hugo")
	defer os.RemoveAll(dir)

	myContentPath := filepath.Join(dir, "mycontent")
	_ = os.Mkdir(myContentPath, os.ModePerm)
	myContent2Path := filepath.Join(dir, "mycontent2")
	_ = os.Mkdir(myContent2Path, os.ModePerm)
	themePath := filepath.Join(dir, "mytheme")
	_ = os.Mkdir(themePath, os.ModePerm)

	mcs := "-- a.md --\n" +
		"mycontent: a\n" +
		"-- c.md --\n" +
		"mycontent: c"
	writeFiles(mcs, myContentPath)

	mc2s := "-- a.md --\n" +
		"mycontent2: a\n" +
		"-- d.md --\n" +
		"mycontent2: d"
	writeFiles(mc2s, myContent2Path)

	ts := "-- a.md --\n" +
		"mytheme: a\n" +
		"-- b.md --\n" +
		"mytheme: b"

	writeFiles(ts, themePath)

	p := &Paths{
		WorkingDir: dir,
		AllModules: []Module{
			{
				ProjectMod: true,
				Dir:        dir,
				Mounts: []Mount{
					{Source: "mycontent",
						Target: "content"},
					{Source: "mycontent2",
						Target: "content"},
				},
			},
			{
				ProjectMod: false,
				Dir:        dir,
				Mounts: []Mount{
					{Source: "mytheme",
						Target: "content"},
				},
			},
		},
	}

	collector := &filesystemsCollector{
		overlayMountsContent: overlayfs.New(
			[]overlayfs.AbsStatFss{}),
	}
	createOverlayFs(collector, p)

	var f fs.File
	fis, _ := collector.
		overlayMountsContent.ReadDir(filepathSeparator)
	for _, fi := range fis {
		fmt.Println(fi.Name())
		f, _ = collector.
			overlayMountsContent.Open(fi.Name())
		b, _ := io.ReadAll(f)
		fmt.Println(string(b))
	}
	defer f.Close()
}

func writeFiles(s string, dir string) {
	data := txtar.Parse([]byte(s))

	for _, f := range data.Files {
		if err := os.WriteFile(
			filepath.Join(dir, f.Name),
			bytes.TrimSuffix(f.Data, []byte("\n")),
			os.ModePerm); err != nil {
			panic(err)
		}
	}
}

var filepathSeparator = string(filepath.Separator)

// RootMapping describes a virtual file or directory mount.
type RootMapping struct {
	// The virtual mount.
	From string
	// The source directory or file.
	To string
	// The base of To. May be empty if an
	// absolute path was provided.
	ToBasedir string
	// Whether this is a mount in the main project.
	IsProject bool
	// The virtual mount point, e.g. "blog".
	path string
}

type Mount struct {
	Source string
	Target string
}

type Module struct {
	ProjectMod bool
	Mounts     []Mount
	Dir        string
}

type Modules []Module

type Paths struct {
	AllModules Modules
	WorkingDir string
}

// A RootMappingFs maps several roots into one.
// Note that the root of this filesystem
// is directories only, and they will be returned
// in Readdir and Readdirnames
// in the order given.
type RootMappingFs struct {
	fs            overlayfs.AbsStatFss
	rootMapToReal *radixtree.Tree
}

type filesystemsCollector struct {
	overlayMountsContent *overlayfs.OverlayFs
}

func createOverlayFs(
	collector *filesystemsCollector,
	path *Paths) {

	for _, md := range path.AllModules {
		var fromToContent []RootMapping
		for _, mount := range md.Mounts {
			rm := RootMapping{
				From:      mount.Target, // content
				To:        mount.Source, // mycontent
				ToBasedir: md.Dir,
				IsProject: md.ProjectMod,
			}
			fromToContent = append(fromToContent, rm)
		}
		rmfsContent := newRootMappingFs(fromToContent...)
		collector.overlayMountsContent = collector.
			overlayMountsContent.Append(rmfsContent)
	}
	return
}

// NewRootMappingFs creates a new RootMappingFs
// on top of the provided with root mappings with
// some optional metadata about the root.
// Note that From represents a virtual root
// that maps to the actual filename in To.
func newRootMappingFs(
	rms ...RootMapping) *RootMappingFs {
	t := radixtree.New()
	var virtualRoots []RootMapping

	for _, rm := range rms {
		key := filepathSeparator + rm.From
		mappings := getRms(t, key)
		mappings = append(mappings, rm)
		t.Insert(key, mappings)

		virtualRoots = append(virtualRoots, rm)
	}

	t.Insert(filepathSeparator, virtualRoots)

	return &RootMappingFs{
		rootMapToReal: t,
	}
}

func (m *RootMappingFs) Abs(name string) []string {
	mappings := getRms(m.rootMapToReal, name)

	var paths []string
	for _, m := range mappings {
		paths = append(paths, path.Join(
			m.ToBasedir, m.To))
	}
	return paths
}

func (m *RootMappingFs) Fss() []fs.StatFS {
	mappings := getRms(
		m.rootMapToReal, filepathSeparator)

	var fss []fs.StatFS
	for _, m := range mappings {
		fss = append(fss, os.DirFS(
			path.Join(m.ToBasedir, m.To)).(fs.StatFS))
	}
	return fss
}

func getRms(t *radixtree.Tree,
	key string) []RootMapping {
	var mappings []RootMapping
	v, found := t.Get(key)
	if found {
		mappings = v.([]RootMapping)
	}
	return mappings
}

a.md
mycontent: a
b.md
mytheme: b
c.md
mycontent: c
d.md
mycontent2: d


Program exited.