Merge pull request #81 from joshbeard/skip-index

feat: skip index

Author: joshbeard

README.md

@@ -45,6 +45,7 @@ Flags:
   -q, --quiet                   Suppress log output
   -r, --recursive               List files recursively
   -S, --skip strings            A list of files or directories to skip. Comma separated or specified multiple times
+      --skipindex-files strings A list of files that indicate a directory should be skipped for indexing but still included in the parent directory listing. Comma separated or specified multiple times (default [.skipindex])
       --sort-by string          The order for the index page. One of: last_modified, name, natural_name (default "natural_name")
   -s, --source string           REQUIRED. The source directory or S3 URI to list
   -t, --target string           REQUIRED. The target directory or S3 URI to write to
@@ -277,6 +278,11 @@ order: "asc"
 # recursive enables indexing the source recursively.
 recursive: false
 
+# skipindex_files is a list of filenames that, when present in a directory,
+# indicate that the directory should be skipped for indexing but still
+# included in the parent directory's listing.
+skipindex_files: [".skipindex"]
+
 # skips is a list of filenames to skip.
 skips: []
 
@@ -334,22 +340,46 @@ This is useful for:
 - Preventing indexing of directories that contain temporary or build files
 - Selectively controlling which directories appear in your indexes
 
+## Excluding Directories from Indexing but Including in Parent Listing
+
+You can also use the `--skipindex-files` flag to specify files that, when present in a directory, indicate that directory should be excluded from indexing but still appear in the parent directory's listing:
+
+1. Skip generating an index for that directory (same as `--noindex-files`)
+2. **Still include the directory in the parent directory's index** (different from `--noindex-files`)
+3. Skip all subdirectories beneath it (same as `--noindex-files`)
+
+This is particularly useful for directories that already have their own index pages (e.g., generated by a web framework) where you want to link to them from the parent directory, but don't want to overwrite their custom index.
+
 ### Examples
 
 To exclude a directory using the default `.noindex` file:
 
 ```shell
-# Create an empty .noindex file in a directory you want to exclude
+# Create an empty .noindex file in a directory you want to exclude completely
 touch /path/to/directory/.noindex
 ```
 
+To exclude a directory from indexing but still include it in the parent's index:
+
+```shell
+# Create an empty .skipindex file
+touch /path/to/directory/.skipindex
+```
+
 To specify custom noindex filenames:
 
 ```shell
 # Use custom filenames instead of the default .noindex
 web-indexer --source /path/to/directory --target /path/to/output --noindex-files .private,.hidden,DO_NOT_INDEX
 ```
 
+To specify custom skipindex filenames:
+
+```shell
+# Use custom filenames instead of the default .skipindex
+web-indexer --source /path/to/directory --target /path/to/output --skipindex-files index.html,.custom-index
+```
+
 In your YAML configuration:
 
 ```yaml
@@ -359,10 +389,19 @@ noindex_files:
   - .private
   - .hidden
   - DO_NOT_INDEX
+skipindex_files:
+  - index.html
+  - .custom-index
 ```
 
 When the indexer encounters a directory with a noindex file, it will log a message at the INFO level:
 
 ```
 INFO Skipping /path/to/directory (found noindex file .noindex)
 ```
+
+When the indexer encounters a directory with a skipindex file, it will log a message at the INFO level:
+
+```
+INFO Skipping indexing of /path/to/directory (found skipindex file .skipindex), will include in parent directory
+```

action.yml

@@ -28,13 +28,11 @@ inputs:
     required: false
   minify:
     description: Boolean toggling minification of the generated HTML
-  noindex_files:
-    description: >
-      A comma-separated list of filenames that will cause a directory/path to be skipped if present
+  noindex-files:
+    description: 'A list of files that indicate a directory should be skipped'
     required: false
   order:
-    description: The order for the items. One of asc, desc (default "asc")
-    required: false
+    description: 'The order for the items. One of: asc, desc'
   recursive:
     description: Index files recursively
   skip:
@@ -59,6 +57,13 @@ inputs:
     description: 'The Docker image tag to use (e.g., latest, dev-pr123)'
     required: false
     default: 'latest'
+  skipindex-files:
+    description: 'A list of files that indicate a directory should be skipped for indexing but still included in the parent directory listing'
+    required: false
+  image_tag:
+    description: 'The Docker image tag to use (e.g., latest, dev-pr123)'
+    required: false
+    default: 'latest'
 
 runs:
   using: 'docker'
@@ -71,7 +76,8 @@ runs:
     LINK_TO_INDEX: ${{ inputs.link_to_index }}
     LOG_LEVEL: ${{ inputs.log_level }}
     MINIFY: ${{ inputs.minify }}
-    NOINDEX_FILES: ${{ inputs.noindex_files }}
+    NOINDEX_FILES: ${{ inputs.noindex-files }}
+    SKIPINDEX_FILES: ${{ inputs.skipindex-files }}
     ORDER: ${{ inputs.order }}
     RECURSIVE: ${{ inputs.recursive }}
     SKIP: ${{ inputs.skip }}

entrypoint.sh

@@ -13,6 +13,7 @@ cmd="web-indexer"
 [[ -n "$LOG_LEVEL" ]] && cmd="$cmd --log-level \"$LOG_LEVEL\""
 [[ "$MINIFY" == "true" ]] && cmd="$cmd --minify"
 [[ -n "$NOINDEX_FILES" ]] && cmd="$cmd --noindex-files \"$NOINDEX_FILES\""
+[[ -n "$SKIPINDEX_FILES" ]] && cmd="$cmd --skipindex-files \"$SKIPINDEX_FILES\""
 [[ -n "$ORDER" ]] && cmd="$cmd --order \"$ORDER\""
 [[ "$RECURSIVE" == "true" ]] && cmd="$cmd --recursive"
 [[ -n "$SKIP" ]] && cmd="$cmd --skip \"$SKIP\""

internal/webindexer/config.go

@@ -5,27 +5,28 @@ import (
 )
 
 type Config struct {
-	BaseURL       string   `yaml:"base_url"      mapstructure:"base_url"`
-	DateFormat    string   `yaml:"date_format"   mapstructure:"date_format"`
-	DirsFirst     bool     `yaml:"dirs_first"    mapstructure:"dirs_first"`
-	IndexFile     string   `yaml:"index_file"    mapstructure:"index_file"`
-	LinkToIndexes bool     `yaml:"link_to_index" mapstructure:"link_to_index"`
-	LogLevel      string   `yaml:"log_level"     mapstructure:"log_level"`
-	LogFile       string   `yaml:"log_file"      mapstructure:"log_file"`
-	Minify        bool     `yaml:"minify"        mapstructure:"minify"`
-	NoIndexFiles  []string `yaml:"noindex_files" mapstructure:"noindex_files"`
-	Order         string   `yaml:"order"         mapstructure:"order"`
-	Quiet         bool     `yaml:"quiet"         mapstructure:"quiet"`
-	Recursive     bool     `yaml:"recursive"     mapstructure:"recursive"`
-	Skips         []string `yaml:"skips"         mapstructure:"skips"`
-	SortBy        string   `yaml:"sort_by"       mapstructure:"sort_by"`
-	Source        string   `yaml:"source"        mapstructure:"source"`
-	Target        string   `yaml:"target"        mapstructure:"target"`
-	Template      string   `yaml:"template"      mapstructure:"template"`
-	Theme         string   `yaml:"theme"         mapstructure:"theme"`
-	Title         string   `yaml:"title"         mapstructure:"title"`
-	CfgFile       string   `yaml:"-"`
-	BasePath      string   `yaml:"-"`
+	BaseURL        string   `yaml:"base_url"      mapstructure:"base_url"`
+	DateFormat     string   `yaml:"date_format"   mapstructure:"date_format"`
+	DirsFirst      bool     `yaml:"dirs_first"    mapstructure:"dirs_first"`
+	IndexFile      string   `yaml:"index_file"    mapstructure:"index_file"`
+	LinkToIndexes  bool     `yaml:"link_to_index" mapstructure:"link_to_index"`
+	LogLevel       string   `yaml:"log_level"     mapstructure:"log_level"`
+	LogFile        string   `yaml:"log_file"      mapstructure:"log_file"`
+	Minify         bool     `yaml:"minify"        mapstructure:"minify"`
+	NoIndexFiles   []string `yaml:"noindex_files" mapstructure:"noindex_files"`
+	SkipIndexFiles []string `yaml:"skipindex_files" mapstructure:"skipindex_files"`
+	Order          string   `yaml:"order"         mapstructure:"order"`
+	Quiet          bool     `yaml:"quiet"         mapstructure:"quiet"`
+	Recursive      bool     `yaml:"recursive"     mapstructure:"recursive"`
+	Skips          []string `yaml:"skips"         mapstructure:"skips"`
+	SortBy         string   `yaml:"sort_by"       mapstructure:"sort_by"`
+	Source         string   `yaml:"source"        mapstructure:"source"`
+	Target         string   `yaml:"target"        mapstructure:"target"`
+	Template       string   `yaml:"template"      mapstructure:"template"`
+	Theme          string   `yaml:"theme"         mapstructure:"theme"`
+	Title          string   `yaml:"title"         mapstructure:"title"`
+	CfgFile        string   `yaml:"-"`
+	BasePath       string   `yaml:"-"`
 }
 
 type SortBy string

internal/webindexer/local.go

@@ -26,9 +26,20 @@ func (l *LocalBackend) Read(path string) ([]Item, bool, error) {
 
 	// First check for noindex files before processing anything else
 	for _, file := range files {
-		if !file.IsDir() && len(l.cfg.NoIndexFiles) > 0 && contains(l.cfg.NoIndexFiles, file.Name()) {
-			log.Infof("Skipping %s (found noindex file %s)", path, file.Name())
-			return nil, true, nil
+		if !file.IsDir() {
+			// Check for noindex files (skip directory entirely)
+			if len(l.cfg.NoIndexFiles) > 0 && contains(l.cfg.NoIndexFiles, file.Name()) {
+				log.Infof("Skipping %s (found noindex file %s)", path, file.Name())
+				return nil, true, nil
+			}
+
+			// Check for skipindex files (skip indexing but include in parent)
+			if len(l.cfg.SkipIndexFiles) > 0 && contains(l.cfg.SkipIndexFiles, file.Name()) {
+				log.Infof("Skipping indexing of %s (found skipindex file %s), will include in parent directory", path, file.Name())
+				// Return empty items but mark as not having noindex file
+				// This will prevent indexing this directory but still include it in the parent
+				return []Item{}, false, nil
+			}
 		}
 	}
 
@@ -44,23 +55,23 @@ func (l *LocalBackend) Read(path string) ([]Item, bool, error) {
 			return nil, false, fmt.Errorf("unable to stat file %s: %w", file.Name(), err)
 		}
 
-		// If it's a directory, check if it contains a noindex file before adding it
+		// If it's a directory, check if it contains a noindex or skipindex file before adding it
 		if stat.IsDir() {
 			subFiles, err := os.ReadDir(fullPath)
 			if err != nil {
 				return nil, false, fmt.Errorf("unable to read directory %s: %w", fullPath, err)
 			}
 
 			// Skip this directory if it contains a noindex file
-			hasNoIndex := false
+			skipDir := false
 			for _, subFile := range subFiles {
 				if !subFile.IsDir() && len(l.cfg.NoIndexFiles) > 0 && contains(l.cfg.NoIndexFiles, subFile.Name()) {
 					log.Infof("Skipping %s (found noindex file %s)", fullPath, subFile.Name())
-					hasNoIndex = true
+					skipDir = true
 					break
 				}
 			}
-			if hasNoIndex {
+			if skipDir {
 				continue
 			}
 		}
@@ -82,6 +93,15 @@ func (l *LocalBackend) Read(path string) ([]Item, bool, error) {
 	return items, false, nil
 }
 
+func (l *LocalBackend) EnsureDirExists(relativePath string) error {
+	localPath := filepath.Join(l.cfg.Target, relativePath)
+	if err := os.MkdirAll(localPath, 0o750); err != nil {
+		return fmt.Errorf("failed to ensure directory exists %s: %w", localPath, err)
+	}
+	log.Debugf("Ensured directory exists: %s", localPath)
+	return nil
+}
+
 func (l *LocalBackend) Write(data Data, content string) error {
 	prefix := data.RelativePath
 	prefix = strings.TrimPrefix(prefix, l.cfg.BasePath)

internal/webindexer/s3.go

@@ -48,13 +48,22 @@ func (s *S3Backend) Read(prefix string) ([]Item, bool, error) {
 		return nil, false, fmt.Errorf("unable to list S3 objects: %w", err)
 	}
 
-	// First check for noindex files before processing anything else
+	// First check for noindex files or skipindex files before processing anything else
 	for _, content := range resp.Contents {
 		fileName := filepath.Base(*content.Key)
+		// Check for noindex files (skip directory entirely)
 		if len(s.cfg.NoIndexFiles) > 0 && contains(s.cfg.NoIndexFiles, fileName) {
 			log.Infof("Skipping %s/%s (found noindex file %s)", s.bucket, prefix, fileName)
 			return nil, true, nil
 		}
+
+		// Check for skipindex files (skip indexing but include in parent)
+		if len(s.cfg.SkipIndexFiles) > 0 && contains(s.cfg.SkipIndexFiles, fileName) {
+			log.Infof("Skipping indexing of %s/%s (found skipindex file %s), will include in parent directory", s.bucket, prefix, fileName)
+			// Return empty items but mark as not having noindex file
+			// This will prevent indexing this directory but still include it in the parent
+			return []Item{}, false, nil
+		}
 	}
 
 	var items []Item
@@ -94,16 +103,16 @@ func (s *S3Backend) Read(prefix string) ([]Item, bool, error) {
 		}
 
 		// Skip this prefix if it contains a noindex file
-		hasNoIndex := false
+		skipDir := false
 		for _, content := range subResp.Contents {
 			fileName := filepath.Base(*content.Key)
 			if len(s.cfg.NoIndexFiles) > 0 && contains(s.cfg.NoIndexFiles, fileName) {
 				log.Infof("Skipping %s/%s (found noindex file %s)", s.bucket, *commonPrefix.Prefix, fileName)
-				hasNoIndex = true
+				skipDir = true
 				break
 			}
 		}
-		if hasNoIndex {
+		if skipDir {
 			continue
 		}
 
@@ -118,6 +127,14 @@ func (s *S3Backend) Read(prefix string) ([]Item, bool, error) {
 	return items, false, nil
 }
 
+// EnsureDirExists is a no-op for S3 as directories are implicit.
+func (s *S3Backend) EnsureDirExists(relativePath string) error {
+	log.Debugf("EnsureDirExists called for S3 (no-op): %s/%s", s.bucket, relativePath)
+	// S3 directories are created implicitly by object keys.
+	// We could potentially check if the bucket exists here if needed.
+	return nil
+}
+
 func (s *S3Backend) Write(data Data, content string) error {
 	bucket, target := uriToBucketAndPrefix(s.cfg.Target)
 	target = strings.TrimPrefix(target, s.cfg.BasePath)