Fix bug in .gitignore, add back *.go
This commit is contained in:
Pim van Pelt
2
.gitignore
vendored
2
.gitignore
vendored
@@ -1,4 +1,4 @@
|
||||
s3-genindex
|
||||
/s3-genindex
|
||||
|
||||
# Generated index files
|
||||
**/index.html
|
||||
|
||||
286
cmd/s3-genindex/main.go
Normal file
286
cmd/s3-genindex/main.go
Normal file
@@ -0,0 +1,286 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
|
||||
"git.ipng.ch/ipng/s3-genindex/internal/indexgen"
|
||||
)
|
||||
|
||||
// S3Config holds S3 connection configuration
|
||||
type S3Config struct {
|
||||
Endpoint string
|
||||
Bucket string
|
||||
Region string
|
||||
UseSSL bool
|
||||
}
|
||||
|
||||
// parseS3URL parses S3 URL and extracts endpoint and bucket
|
||||
// Example: http://minio0.chbtl0.net.ipng.ch:9000/ctlog-ro
|
||||
// Returns: endpoint=minio0.chbtl0.net.ipng.ch:9000, bucket=ctlog-ro, useSSL=false
|
||||
func parseS3URL(s3URL string) (*S3Config, error) {
|
||||
u, err := url.Parse(s3URL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse URL: %w", err)
|
||||
}
|
||||
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return nil, fmt.Errorf("unsupported scheme: %s", u.Scheme)
|
||||
}
|
||||
|
||||
// Extract bucket from path
|
||||
path := strings.Trim(u.Path, "/")
|
||||
if path == "" {
|
||||
return nil, fmt.Errorf("bucket name not found in URL path")
|
||||
}
|
||||
|
||||
// For MinIO/S3 URLs like http://host:port/bucket, the bucket is the first path segment
|
||||
bucket := strings.Split(path, "/")[0]
|
||||
|
||||
config := &S3Config{
|
||||
Endpoint: u.Host,
|
||||
Bucket: bucket,
|
||||
Region: "us-east-1", // Default region
|
||||
UseSSL: u.Scheme == "https",
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// processS3Bucket processes an S3 bucket and generates index files
|
||||
func processS3Bucket(s3Config *S3Config, opts *indexgen.Options) error {
|
||||
// Get credentials from environment variables
|
||||
accessKey := os.Getenv("AWS_ACCESS_KEY_ID")
|
||||
secretKey := os.Getenv("AWS_SECRET_ACCESS_KEY")
|
||||
|
||||
if accessKey == "" || secretKey == "" {
|
||||
return fmt.Errorf("AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables must be set")
|
||||
}
|
||||
|
||||
// Create AWS config with custom endpoint and credentials
|
||||
cfg := aws.Config{
|
||||
Region: s3Config.Region,
|
||||
Credentials: credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),
|
||||
}
|
||||
|
||||
// Create S3 client with custom endpoint resolver
|
||||
client := s3.NewFromConfig(cfg, func(o *s3.Options) {
|
||||
o.BaseEndpoint = aws.String(fmt.Sprintf("%s://%s",
|
||||
map[bool]string{true: "https", false: "http"}[s3Config.UseSSL],
|
||||
s3Config.Endpoint))
|
||||
o.UsePathStyle = true // Use path-style URLs for MinIO compatibility
|
||||
})
|
||||
|
||||
// List objects in the bucket
|
||||
ctx := context.Background()
|
||||
input := &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(s3Config.Bucket),
|
||||
}
|
||||
|
||||
if opts.Verbose {
|
||||
log.Printf("Listing objects in S3 bucket: %s", s3Config.Bucket)
|
||||
}
|
||||
|
||||
result, err := client.ListObjectsV2(ctx, input)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list S3 objects: %w", err)
|
||||
}
|
||||
|
||||
// Convert S3 objects to FileEntry format
|
||||
var entries []indexgen.FileEntry
|
||||
for _, obj := range result.Contents {
|
||||
if obj.Key == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
keyName := *obj.Key
|
||||
|
||||
// Skip if excluded by regex
|
||||
if opts.ExcludeRegex != nil && opts.ExcludeRegex.MatchString(keyName) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip hidden files if not included
|
||||
if !opts.IncludeHidden && strings.HasPrefix(keyName, ".") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Simple glob matching for filter
|
||||
if opts.Filter != "*" && opts.Filter != "" {
|
||||
matched, err := filepath.Match(opts.Filter, keyName)
|
||||
if err != nil || !matched {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
entry := indexgen.FileEntry{
|
||||
Name: keyName,
|
||||
Path: keyName,
|
||||
IsDir: false,
|
||||
Size: *obj.Size,
|
||||
ModTime: *obj.LastModified,
|
||||
IsSymlink: false,
|
||||
IconType: indexgen.GetIconType(keyName),
|
||||
SizePretty: indexgen.PrettySize(*obj.Size),
|
||||
ModTimeISO: obj.LastModified.Format("2006-01-02T15:04:05Z"),
|
||||
}
|
||||
|
||||
// Set CSS class based on file type
|
||||
if entry.IsDir {
|
||||
entry.CSSClass = "dir"
|
||||
} else if entry.IsSymlink {
|
||||
entry.CSSClass = "symlink"
|
||||
} else {
|
||||
entry.CSSClass = "file"
|
||||
}
|
||||
|
||||
entries = append(entries, entry)
|
||||
|
||||
if opts.Verbose {
|
||||
log.Printf("Found object: %s (%s)", entry.Name, entry.SizePretty)
|
||||
}
|
||||
}
|
||||
|
||||
// Set TopDir to bucket name for template generation
|
||||
opts.TopDir = s3Config.Bucket
|
||||
|
||||
// Generate HTML from entries - need to implement this function
|
||||
return generateS3HTML(entries, opts)
|
||||
}
|
||||
|
||||
// generateS3HTML generates HTML index for S3 objects using the existing template system
|
||||
func generateS3HTML(entries []indexgen.FileEntry, opts *indexgen.Options) error {
|
||||
// Sort entries by name (similar to filesystem behavior)
|
||||
sort.Slice(entries, func(i, j int) bool {
|
||||
return entries[i].Name < entries[j].Name
|
||||
})
|
||||
|
||||
// Get the HTML template
|
||||
tmpl := indexgen.GetHTMLTemplate()
|
||||
if tmpl == nil {
|
||||
return fmt.Errorf("failed to get HTML template")
|
||||
}
|
||||
|
||||
// Prepare template data (similar to ProcessDir in indexgen)
|
||||
data := struct {
|
||||
DirName string
|
||||
Entries []indexgen.FileEntry
|
||||
TopDir string
|
||||
Hostname string
|
||||
}{
|
||||
DirName: opts.TopDir, // Use bucket name as directory name
|
||||
Entries: entries,
|
||||
TopDir: opts.TopDir,
|
||||
Hostname: "S3 Bucket", // Could be improved to show actual endpoint
|
||||
}
|
||||
|
||||
// Determine output file
|
||||
outputFile := opts.OutputFile
|
||||
if outputFile == "" {
|
||||
outputFile = indexgen.DefaultOutputFile
|
||||
}
|
||||
|
||||
// Create output file
|
||||
file, err := os.Create(outputFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create output file %s: %w", outputFile, err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Execute template
|
||||
err = tmpl.Execute(file, data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to execute template: %w", err)
|
||||
}
|
||||
|
||||
if opts.Verbose {
|
||||
log.Printf("Generated index file: %s (%d entries)", outputFile, len(entries))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
var opts indexgen.Options
|
||||
var excludeRegexStr string
|
||||
var s3Mode bool
|
||||
var directory string
|
||||
|
||||
// Set defaults
|
||||
opts.DirAppend = true
|
||||
opts.OutputFile = indexgen.DefaultOutputFile
|
||||
opts.Recursive = true
|
||||
opts.IncludeHidden = true
|
||||
|
||||
flag.StringVar(&directory, "d", "", "directory or S3 URL to process (required)")
|
||||
flag.StringVar(&opts.Filter, "f", "*", "only include files matching glob")
|
||||
flag.StringVar(&excludeRegexStr, "x", "", "exclude files matching regular expression")
|
||||
flag.BoolVar(&opts.Verbose, "v", false, "verbosely list every processed file")
|
||||
flag.BoolVar(&s3Mode, "s3", false, "treat -d argument as S3 URL")
|
||||
|
||||
flag.Usage = func() {
|
||||
fmt.Fprintf(os.Stderr, "Generate directory index files (recursive is ON, hidden files included by default).\n")
|
||||
fmt.Fprintf(os.Stderr, "Output file is 'index.html', directory href appending is enabled.\n")
|
||||
fmt.Fprintf(os.Stderr, "Directory must be specified with -d flag.\n")
|
||||
fmt.Fprintf(os.Stderr, "For S3 mode, use -s3 flag with S3 URL in -d.\n")
|
||||
fmt.Fprintf(os.Stderr, "S3 URLs: http://host:port/bucket or https://host/bucket\n")
|
||||
fmt.Fprintf(os.Stderr, "For S3, set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.\n")
|
||||
fmt.Fprintf(os.Stderr, "Optionally filter by file types with -f \"*.py\".\n\n")
|
||||
fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS]\n\n", os.Args[0])
|
||||
fmt.Fprintf(os.Stderr, "Examples:\n")
|
||||
fmt.Fprintf(os.Stderr, " %s -d /path/to/dir\n", os.Args[0])
|
||||
fmt.Fprintf(os.Stderr, " %s -d http://minio.example.com:9000/bucket -s3\n", os.Args[0])
|
||||
fmt.Fprintf(os.Stderr, " %s -v -f \"*.log\" -d https://s3.amazonaws.com/logs -s3\n\n", os.Args[0])
|
||||
flag.PrintDefaults()
|
||||
}
|
||||
|
||||
flag.Parse()
|
||||
|
||||
// Check if directory flag is provided
|
||||
if directory == "" {
|
||||
fmt.Fprintf(os.Stderr, "Error: Directory must be specified with -d flag.\n\n")
|
||||
flag.Usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if excludeRegexStr != "" {
|
||||
var err error
|
||||
opts.ExcludeRegex, err = regexp.Compile(excludeRegexStr)
|
||||
if err != nil {
|
||||
log.Fatal("Invalid regular expression:", err)
|
||||
}
|
||||
}
|
||||
|
||||
if s3Mode {
|
||||
// Parse S3 URL
|
||||
s3Config, err := parseS3URL(directory)
|
||||
if err != nil {
|
||||
log.Fatal("Failed to parse S3 URL:", err)
|
||||
}
|
||||
|
||||
// Process S3 bucket
|
||||
err = processS3Bucket(s3Config, &opts)
|
||||
if err != nil {
|
||||
log.Fatal("Failed to process S3 bucket:", err)
|
||||
}
|
||||
} else {
|
||||
// Process local directory
|
||||
opts.TopDir = directory
|
||||
err := indexgen.ProcessDir(opts.TopDir, &opts)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
282
cmd/s3-genindex/main_test.go
Normal file
282
cmd/s3-genindex/main_test.go
Normal file
@@ -0,0 +1,282 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// resetFlags resets command line flags for testing
|
||||
func resetFlags() {
|
||||
flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
|
||||
}
|
||||
|
||||
func TestMainWithHelp(t *testing.T) {
|
||||
// Save original args
|
||||
oldArgs := os.Args
|
||||
|
||||
// Test help flag
|
||||
os.Args = []string{"s3-genindex", "--help"}
|
||||
|
||||
// Reset flags
|
||||
resetFlags()
|
||||
|
||||
// We can't easily test main() directly since it calls os.Exit,
|
||||
// but we can test that the usage function works
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
// Expected to panic/exit on --help
|
||||
t.Log("Help flag caused expected panic/exit")
|
||||
}
|
||||
os.Args = oldArgs
|
||||
}()
|
||||
|
||||
// This would normally exit, but we're just testing that it doesn't crash
|
||||
}
|
||||
|
||||
func TestMainWithVersion(t *testing.T) {
|
||||
oldArgs := os.Args
|
||||
defer func() { os.Args = oldArgs }()
|
||||
|
||||
// Test that main doesn't crash with basic arguments
|
||||
os.Args = []string{"s3-genindex"}
|
||||
|
||||
// We can't easily test main() execution without it creating files,
|
||||
// so this test just ensures the package compiles correctly
|
||||
}
|
||||
|
||||
func TestFlagParsing(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
}{
|
||||
{
|
||||
name: "default flags",
|
||||
args: []string{"s3-genindex"},
|
||||
},
|
||||
{
|
||||
name: "with directory flag",
|
||||
args: []string{"s3-genindex", "-d", "/tmp"},
|
||||
},
|
||||
{
|
||||
name: "verbose flag with directory",
|
||||
args: []string{"s3-genindex", "-v", "-d", "/tmp"},
|
||||
},
|
||||
{
|
||||
name: "exclude regex flag with directory",
|
||||
args: []string{"s3-genindex", "-x", "*.tmp", "-d", "/tmp"},
|
||||
},
|
||||
{
|
||||
name: "multiple flags with directory",
|
||||
args: []string{"s3-genindex", "-v", "-x", "*.tmp", "-d", "/tmp"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Save original args
|
||||
oldArgs := os.Args
|
||||
defer func() { os.Args = oldArgs }()
|
||||
|
||||
// Set test args
|
||||
os.Args = tt.args
|
||||
|
||||
// Reset flags for each test
|
||||
resetFlags()
|
||||
|
||||
// Test that flag parsing doesn't panic
|
||||
// Note: We're not actually calling main() here since that would
|
||||
// try to create files, but this tests that our flag setup is correct
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidRegex(t *testing.T) {
|
||||
oldArgs := os.Args
|
||||
defer func() { os.Args = oldArgs }()
|
||||
|
||||
// Test invalid regex - this would normally cause log.Fatal
|
||||
os.Args = []string{"s3-genindex", "-x", "[invalid"}
|
||||
|
||||
resetFlags()
|
||||
|
||||
// We can't easily test log.Fatal without refactoring main(),
|
||||
// but this ensures the test setup is correct
|
||||
}
|
||||
|
||||
// Test that main package compiles correctly
|
||||
func TestMainCompiles(t *testing.T) {
|
||||
// This test just ensures the main package compiles without issues
|
||||
// More comprehensive integration testing is done in the indexgen package
|
||||
}
|
||||
|
||||
func TestS3FlagHandling(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
}{
|
||||
{
|
||||
name: "s3 flag with URL",
|
||||
args: []string{"s3-genindex", "-s3", "-d", "http://minio.example.com:9000/bucket"},
|
||||
},
|
||||
{
|
||||
name: "s3 flag with verbose",
|
||||
args: []string{"s3-genindex", "-v", "-s3", "-d", "https://s3.amazonaws.com/logs"},
|
||||
},
|
||||
{
|
||||
name: "local directory without s3 flag",
|
||||
args: []string{"s3-genindex", "-d", "/tmp"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Save original args
|
||||
oldArgs := os.Args
|
||||
defer func() { os.Args = oldArgs }()
|
||||
|
||||
// Set test args
|
||||
os.Args = tt.args
|
||||
|
||||
// Reset flags for each test
|
||||
resetFlags()
|
||||
|
||||
// Test that flag parsing doesn't panic
|
||||
// Note: We're not actually calling main() here since that would
|
||||
// try to create files or connect to S3, but this tests that our flag setup is correct
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMandatoryDirectoryArgument(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
shouldExitWithError bool
|
||||
}{
|
||||
{
|
||||
name: "no directory flag",
|
||||
args: []string{"s3-genindex"},
|
||||
shouldExitWithError: true,
|
||||
},
|
||||
{
|
||||
name: "with directory flag",
|
||||
args: []string{"s3-genindex", "-d", "/tmp"},
|
||||
shouldExitWithError: false,
|
||||
},
|
||||
{
|
||||
name: "with s3 flag but no directory",
|
||||
args: []string{"s3-genindex", "-s3"},
|
||||
shouldExitWithError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Save original args
|
||||
oldArgs := os.Args
|
||||
defer func() { os.Args = oldArgs }()
|
||||
|
||||
// Set test args
|
||||
os.Args = tt.args
|
||||
|
||||
// Reset flags for each test
|
||||
resetFlags()
|
||||
|
||||
// We can't easily test os.Exit calls without refactoring,
|
||||
// but this ensures the test setup is correct
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseS3URL(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
url string
|
||||
expected *S3Config
|
||||
hasError bool
|
||||
}{
|
||||
{
|
||||
name: "MinIO HTTP URL",
|
||||
url: "http://minio0.chbtl0.net.ipng.ch:9000/ctlog-ro",
|
||||
expected: &S3Config{
|
||||
Endpoint: "minio0.chbtl0.net.ipng.ch:9000",
|
||||
Bucket: "ctlog-ro",
|
||||
Region: "us-east-1",
|
||||
UseSSL: false,
|
||||
},
|
||||
hasError: false,
|
||||
},
|
||||
{
|
||||
name: "AWS S3 HTTPS URL",
|
||||
url: "https://s3.amazonaws.com/my-bucket",
|
||||
expected: &S3Config{
|
||||
Endpoint: "s3.amazonaws.com",
|
||||
Bucket: "my-bucket",
|
||||
Region: "us-east-1",
|
||||
UseSSL: true,
|
||||
},
|
||||
hasError: false,
|
||||
},
|
||||
{
|
||||
name: "URL with path prefix",
|
||||
url: "http://localhost:9000/bucket/folder/file",
|
||||
expected: &S3Config{
|
||||
Endpoint: "localhost:9000",
|
||||
Bucket: "bucket",
|
||||
Region: "us-east-1",
|
||||
UseSSL: false,
|
||||
},
|
||||
hasError: false,
|
||||
},
|
||||
{
|
||||
name: "Invalid URL",
|
||||
url: "not-a-url",
|
||||
expected: nil,
|
||||
hasError: true,
|
||||
},
|
||||
{
|
||||
name: "URL without bucket",
|
||||
url: "http://minio.example.com:9000/",
|
||||
expected: nil,
|
||||
hasError: true,
|
||||
},
|
||||
{
|
||||
name: "Unsupported scheme",
|
||||
url: "ftp://example.com/bucket",
|
||||
expected: nil,
|
||||
hasError: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result, err := parseS3URL(tt.url)
|
||||
|
||||
if tt.hasError {
|
||||
if err == nil {
|
||||
t.Errorf("parseS3URL(%q) expected error, got nil", tt.url)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
t.Errorf("parseS3URL(%q) unexpected error: %v", tt.url, err)
|
||||
return
|
||||
}
|
||||
|
||||
if result.Endpoint != tt.expected.Endpoint {
|
||||
t.Errorf("parseS3URL(%q) Endpoint = %q, want %q", tt.url, result.Endpoint, tt.expected.Endpoint)
|
||||
}
|
||||
if result.Bucket != tt.expected.Bucket {
|
||||
t.Errorf("parseS3URL(%q) Bucket = %q, want %q", tt.url, result.Bucket, tt.expected.Bucket)
|
||||
}
|
||||
if result.Region != tt.expected.Region {
|
||||
t.Errorf("parseS3URL(%q) Region = %q, want %q", tt.url, result.Region, tt.expected.Region)
|
||||
}
|
||||
if result.UseSSL != tt.expected.UseSSL {
|
||||
t.Errorf("parseS3URL(%q) UseSSL = %v, want %v", tt.url, result.UseSSL, tt.expected.UseSSL)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user