Fix s3 file+dir handling

2025-12-03 12:14:13 +01:00
parent 2274372119
commit 11fbbd4b42
2 changed files with 162 additions and 69 deletions
--- a/cmd/s3-genindex/main.go
+++ b/cmd/s3-genindex/main.go
@@ -28,6 +28,13 @@ type S3Config struct {
 	UseSSL   bool
 }

+// S3Object represents an S3 object
+type S3Object struct {
+	Key          string
+	Size         int64
+	LastModified time.Time
+}
+
 // parseS3URL parses S3 URL and extracts endpoint and bucket
 // Example: http://minio0.chbtl0.net.ipng.ch:9000/ctlog-ro
 // Returns: endpoint=minio0.chbtl0.net.ipng.ch:9000, bucket=ctlog-ro, useSSL=false
@@ -62,15 +69,6 @@ func parseS3URL(s3URL string) (*S3Config, error) {

 // processS3Bucket processes an S3 bucket and generates index files
 func processS3Bucket(s3Config *S3Config, opts *indexgen.Options) error {
-	if opts.DryRun {
-		// In dry run mode, just show what would be done without connecting
-		fmt.Printf("Would connect to S3 endpoint: %s\n", s3Config.Endpoint)
-		fmt.Printf("Would list objects in bucket: %s\n", s3Config.Bucket)
-		fmt.Printf("Would write S3 index file: %s\n", opts.OutputFile)
-		fmt.Printf("Note: Dry run mode - no actual S3 connection made\n")
-		return nil
-	}
-
 	// Get credentials from environment variables
 	accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
 	secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
@@ -108,8 +106,8 @@ func processS3Bucket(s3Config *S3Config, opts *indexgen.Options) error {
 		return fmt.Errorf("failed to list S3 objects: %w", err)
 	}

-	// Convert S3 objects to FileEntry format
-	var entries []indexgen.FileEntry
+	// Collect all S3 objects
+	var allObjects []S3Object
 	for _, obj := range result.Contents {
 		if obj.Key == nil {
 			continue
@@ -135,67 +133,136 @@ func processS3Bucket(s3Config *S3Config, opts *indexgen.Options) error {
 			}
 		}

-		entry := indexgen.FileEntry{
-			Name:         keyName,
-			Path:         keyName,
-			IsDir:        false,
+		allObjects = append(allObjects, S3Object{
+			Key:          keyName,
 			Size:         *obj.Size,
-			ModTime:      *obj.LastModified,
-			IsSymlink:    false,
-			IconType:     indexgen.GetIconType(keyName),
-			SizePretty:   indexgen.PrettySize(*obj.Size),
-			ModTimeISO:   obj.LastModified.Format(time.RFC3339),
-			ModTimeHuman: obj.LastModified.Format(time.RFC822),
-		}
-
-		// Set CSS class based on file type
-		if entry.IsDir {
-			entry.CSSClass = "dir"
-		} else if entry.IsSymlink {
-			entry.CSSClass = "symlink"
-		} else {
-			entry.CSSClass = "file"
-		}
-
-		entries = append(entries, entry)
+			LastModified: *obj.LastModified,
+		})

 		if opts.Verbose {
-			log.Printf("Found object: %s (%s)", entry.Name, entry.SizePretty)
+			log.Printf("Found object: %s (%s)", keyName, indexgen.PrettySize(*obj.Size))
 		}
 	}

-	// Set TopDir to bucket name for template generation
-	opts.TopDir = s3Config.Bucket
-
-	// Generate HTML from entries - need to implement this function
-	return generateS3HTML(entries, opts)
+	// Process hierarchical directory structure
+	return processS3Hierarchy(allObjects, opts, client, s3Config)
 }

-// generateS3HTML generates HTML index for S3 objects using the existing template system
-func generateS3HTML(entries []indexgen.FileEntry, opts *indexgen.Options) error {
+// processS3Hierarchy processes S3 objects hierarchically like filesystem directories
+func processS3Hierarchy(objects []S3Object, opts *indexgen.Options, client *s3.Client, s3Config *S3Config) error {
+	// Group objects by directory path
+	dirMap := make(map[string][]indexgen.FileEntry)
+	
+	// Track all directory paths we need to create indexes for
+	allDirs := make(map[string]bool)
+	
+	for _, obj := range objects {
+		// Split the key into directory parts
+		parts := strings.Split(obj.Key, "/")
+		
+		if len(parts) == 1 {
+			// Root level file
+			entry := createFileEntry(obj, obj.Key)
+			dirMap[""] = append(dirMap[""], entry)
+		} else {
+			// File in a subdirectory
+			fileName := parts[len(parts)-1]
+			dirPath := strings.Join(parts[:len(parts)-1], "/")
+			
+			// Create file entry
+			entry := createFileEntry(obj, fileName)
+			dirMap[dirPath] = append(dirMap[dirPath], entry)
+			
+			// Track all parent directories
+			currentPath := ""
+			for i, part := range parts[:len(parts)-1] {
+				if i == 0 {
+					currentPath = part
+				} else {
+					currentPath = currentPath + "/" + part
+				}
+				allDirs[currentPath] = true
+			}
+		}
+	}
+	
+	// Add directory entries to parent directories
+	for dirPath := range allDirs {
+		parentPath := ""
+		if strings.Contains(dirPath, "/") {
+			parts := strings.Split(dirPath, "/")
+			parentPath = strings.Join(parts[:len(parts)-1], "/")
+		}
+		
+		dirName := filepath.Base(dirPath)
+		// Build the correct path for S3
+		dirEntryPath := dirPath + "/"
+		if opts.DirAppend {
+			dirEntryPath += opts.OutputFile
+		}
+		
+		dirEntry := indexgen.FileEntry{
+			Name:         dirName,
+			Path:         dirEntryPath,
+			IsDir:        true,
+			Size:         -1,
+			IsSymlink:    false,
+			IconType:     "folder",
+			CSSClass:     "folder_filled",
+			SizePretty:   "&mdash;",
+			ModTimeISO:   time.Now().Format(time.RFC3339),
+			ModTimeHuman: time.Now().Format(time.RFC822),
+		}
+		
+		dirMap[parentPath] = append(dirMap[parentPath], dirEntry)
+	}
+	
+	// Set TopDir to bucket name for template generation
+	opts.TopDir = s3Config.Bucket
+	
+	// Generate index.html for each directory
+	for dirPath, entries := range dirMap {
+		indexKey := dirPath
+		if indexKey != "" {
+			indexKey += "/"
+		}
+		indexKey += opts.OutputFile
+		
+		err := generateS3HTML(entries, opts, client, s3Config, indexKey)
+		if err != nil {
+			return fmt.Errorf("failed to generate index for %s: %w", dirPath, err)
+		}
+	}
+	
+	return nil
+}
+
+// createFileEntry creates a FileEntry from an S3Object
+func createFileEntry(obj S3Object, displayName string) indexgen.FileEntry {
+	return indexgen.FileEntry{
+		Name:         displayName,
+		Path:         displayName,
+		IsDir:        false,
+		Size:         obj.Size,
+		ModTime:      obj.LastModified,
+		IsSymlink:    false,
+		IconType:     indexgen.GetIconType(displayName),
+		CSSClass:     "file",
+		SizePretty:   indexgen.PrettySize(obj.Size),
+		ModTimeISO:   obj.LastModified.Format(time.RFC3339),
+		ModTimeHuman: obj.LastModified.Format(time.RFC822),
+	}
+}
+
+// generateS3HTML generates HTML index for S3 objects and uploads to S3
+func generateS3HTML(entries []indexgen.FileEntry, opts *indexgen.Options, client *s3.Client, s3Config *S3Config, indexKey string) error {
 	// Sort entries by name (similar to filesystem behavior)
 	sort.Slice(entries, func(i, j int) bool {
 		return entries[i].Name < entries[j].Name
 	})

-	// Determine output file
-	outputFile := opts.OutputFile
-	if outputFile == "" {
-		outputFile = indexgen.DefaultOutputFile
-	}
+	// Use the provided index key

-	if opts.DryRun {
-		// Dry run mode: show what would be written
-		fmt.Printf("Would write S3 index file: %s\n", outputFile)
-		fmt.Printf("S3 bucket: %s\n", opts.TopDir)
-		fmt.Printf("Objects found: %d\n", len(entries))
-		for _, entry := range entries {
-			fmt.Printf("  object: %s (%s)\n", entry.Name, entry.SizePretty)
-		}
-		return nil
-	}
-
-	// Normal mode: actually write the file
 	// Get the HTML template
 	tmpl := indexgen.GetHTMLTemplate()
 	if tmpl == nil {
@@ -215,21 +282,47 @@ func generateS3HTML(entries []indexgen.FileEntry, opts *indexgen.Options) error
 		OutputFile: opts.OutputFile,
 	}

-	// Create output file
-	file, err := os.Create(outputFile)
-	if err != nil {
-		return fmt.Errorf("failed to create output file %s: %w", outputFile, err)
-	}
-	defer file.Close()
-
-	// Execute template
-	err = tmpl.Execute(file, data)
+	// Generate HTML content in memory
+	var htmlBuffer strings.Builder
+	err := tmpl.Execute(&htmlBuffer, data)
 	if err != nil {
 		return fmt.Errorf("failed to execute template: %w", err)
 	}
+	
+	htmlContent := htmlBuffer.String()
+
+	if opts.DryRun {
+		// Dry run mode: show what would be written but don't upload
+		fmt.Printf("Would upload S3 index file: s3://%s/%s\n", s3Config.Bucket, indexKey)
+		fmt.Printf("Directory level: %s\n", strings.TrimSuffix(indexKey, "/"+opts.OutputFile))
+		fmt.Printf("Objects found: %d\n", len(entries))
+		fmt.Printf("Generated HTML size: %d bytes\n", len(htmlContent))
+		for _, entry := range entries {
+			entryType := "file"
+			if entry.IsDir {
+				entryType = "directory"
+			}
+			fmt.Printf("  %s: %s (%s)\n", entryType, entry.Name, entry.SizePretty)
+		}
+		return nil
+	}
+
+	// Upload HTML to S3
+	ctx := context.Background()
+	putInput := &s3.PutObjectInput{
+		Bucket:      aws.String(s3Config.Bucket),
+		Key:         aws.String(indexKey),
+		Body:        strings.NewReader(htmlContent),
+		ContentType: aws.String("text/html"),
+	}
+
+	_, err = client.PutObject(ctx, putInput)
+	if err != nil {
+		return fmt.Errorf("failed to upload %s to S3: %w", indexKey, err)
+	}

 	if opts.Verbose {
-		log.Printf("Generated index file: %s (%d entries)", outputFile, len(entries))
+		log.Printf("Uploaded index file: %s to S3 bucket %s (%d entries)", indexKey, s3Config.Bucket, len(entries))
 	}

 	return nil
--- a/internal/indexgen/indexgen.go
+++ b/internal/indexgen/indexgen.go
@@ -973,7 +973,7 @@ const htmlTemplateString = `<!DOCTYPE html>
        <tr class="file">
            <td></td>
            <td>
-                <a href="{{urlEscape .Path}}">
+                <a href="{{.Path}}">
                    <svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" stroke-width="2" stroke="currentColor" fill="none" stroke-linecap="round" stroke-linejoin="round"><use xlink:href="#{{.IconType}}" class="{{.CSSClass}}"></use></svg>
                    <span class="name">{{.Name}}</span>
                </a>