Release v1.3.2

debian/changelog

@@ -1,3 +1,26 @@
+ipng-router-backup (1.3.2) stable; urgency=low
+
+  * Fix --key-file authentication priority issue
+  * Prioritize explicit key file over SSH agent authentication
+
+ -- Pim van Pelt <pim@ipng.ch>  Sun, 13 Jul 2025 23:30:00 +0100
+
+ipng-router-backup (1.3.1) stable; urgency=low
+
+  * Fix golangci-lint issues, replace deprecated io/ioutil
+  * Add SSH key error messages with hostname prefix
+  * Independently validate sshkey, agent auth and password methods
+
+ -- Pim van Pelt <pim@ipng.ch>  Sun, 07 Jul 2025 23:30:00 +0100
+
+ipng-router-backup (1.3.0) stable; urgency=low
+
+  * Add --parallel flag for concurrent device processing (default: 10)
+  * Implement worker pool pattern for much faster multi-device backups
+  * Maintain atomic file operations and error handling in parallel mode
+
+ -- Pim van Pelt <pim@ipng.ch>  Sun, 07 Jul 2025 23:00:00 +0100
+
 ipng-router-backup (1.2.4) stable; urgency=low
 
   * Add regex exclude patterns to filter unwanted output lines per device type

docs/DETAILS.md

@@ -34,6 +34,14 @@ types:
       - show version
       - show inventory
       - show running-config
+
+  routeros:
+    commands:
+      - system package print detail without-paging
+      - / export terse
+    exclude:
+      - "^# ....-..-.. ..:..:.. by RouterOS"    # Filter timestamp headers
+      - "^# .../../.... ..:..:.. by RouterOS"   # Alternative date format
 ```
 
 **Main configuration** (`config.yaml`):
@@ -64,6 +72,7 @@ devices:
 #### Types Section
 - **`<type-name>`**: Device type name (e.g., `srlinux`, `eos`)
   - **`commands`**: Array of CLI commands to execute
+  - **`exclude`** (optional): Array of regex patterns to filter out unwanted lines from output
 
 #### Devices Section
 - **`<hostname>`**: Device hostname (used for SSH config lookup and output filename)
@@ -85,6 +94,43 @@ ipng-router-backup --yaml "*.yaml"
 ipng-router-backup --yaml "config/*.yaml"
 ```
 
+## Output Filtering
+
+The tool supports filtering unwanted lines from command output using regular expressions in the `exclude` field of device types.
+
+### How Exclude Patterns Work
+
+- **Regex matching**: Each line of command output is tested against all exclude patterns
+- **Line removal**: Lines matching any pattern are completely removed from the output file
+- **Per-device type**: Exclude patterns are defined at the device type level and apply to all devices of that type
+
+### Common Use Cases
+
+```yaml
+types:
+  routeros:
+    commands:
+      - / export terse
+    exclude:
+      - "^# ....-..-.. ..:..:.. by RouterOS"    # Remove timestamp headers
+      - "^# .../../.... ..:..:.. by RouterOS"   # Alternative date format
+
+  cisco-ios:
+    commands:
+      - show running-config
+    exclude:
+      - "^Building configuration"               # Remove config build messages
+      - "^Current configuration"                # Remove current config headers
+      - "^!"                                   # Remove comment lines
+
+  debug-device:
+    commands:
+      - show logs
+    exclude:
+      - "^DEBUG:"                             # Filter debug messages
+      - "^TRACE:"                             # Filter trace messages
+```
+
 ## Command Line Usage
 
 ### Required Flags
@@ -96,6 +142,7 @@ ipng-router-backup --yaml "config/*.yaml"
 - **`--password`**: SSH password
 - **`--key-file`**: SSH private key file path
 - **`--port`**: SSH port (default: `22`)
+- **`--parallel`**: Maximum number of devices to process in parallel (default: `10`)
 
 ### Examples
 
@@ -114,6 +161,9 @@ ipng-router-backup --yaml config.yaml --output-dir /backup/network
 
 # With password authentication
 ipng-router-backup --yaml config.yaml --password mypassword
+
+# Process more devices in parallel
+ipng-router-backup --yaml config.yaml --parallel 20
 ```
 
 ## SSH Authentication

docs/ipng-router-backup.1

@@ -35,6 +35,9 @@ SSH port number (default: 22)
 .BR --host " \fIHOSTNAME\fR"
 Specific host(s) or glob patterns to process (can be repeated, processes all if not specified)
 .TP
+.BR --parallel " \fINUMBER\fR"
+Maximum number of devices to process in parallel (default: 10)
+.TP
 .BR --help
 Show help message
 .SH CONFIGURATION
@@ -48,6 +51,11 @@ types:
     commands:
       - show version
       - show platform linecard
+  routeros:
+    commands:
+      - / export terse
+    exclude:
+      - "^# ....-..-.. ..:..:.. by RouterOS"
 .EE
 .SS devices
 Define individual devices:
@@ -70,6 +78,8 @@ Default SSH keys (~/.ssh/id_rsa, ~/.ssh/id_ed25519, ~/.ssh/id_ecdsa)
 Password authentication (--password option)
 .SH OUTPUT
 For each device, a text file named after the hostname is created in the specified directory. Each command output is prefixed with "## COMMAND: <command_name>" for easy identification.
+.PP
+Output can be filtered using regex patterns defined in the device type's 'exclude' field to remove unwanted lines such as timestamps or debug messages.
 .SH EXAMPLES
 .TP
 Basic usage with glob patterns:
@@ -91,6 +101,11 @@ Process hosts matching patterns:
 .EX
 ipng-router-backup --yaml config.yaml --host "asw*" --host "*switch*"
 .EE
+.TP
+Process devices in parallel:
+.EX
+ipng-router-backup --yaml config.yaml --parallel 20
+.EE
 .SH FILES
 .TP
 .I /etc/ipng-router-backup/config.yaml.example

src/config.go

@@ -4,7 +4,7 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
+	"os"
 
 	"dario.cat/mergo"
 	"gopkg.in/yaml.v3"
@@ -29,7 +29,7 @@ type Device struct {
 }
 
 func readYAMLFile(path string) (map[string]interface{}, error) {
-	data, err := ioutil.ReadFile(path)
+	data, err := os.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}

src/main.go

@@ -7,19 +7,34 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"sync"
 
 	"github.com/spf13/cobra"
 )
 
-const Version = "1.2.4"
+const Version = "1.3.2"
 
-// Config and SSH types are now in separate packages
+func processDevice(hostname string, deviceConfig Device, commands []string, excludePatterns []string, password, keyFile string, port int, outputDir string) bool {
+	// Create backup instance
+	backup := NewRouterBackup(hostname, deviceConfig.Address, deviceConfig.User, password, keyFile, port)
 
-// SSH connection methods are now in ssh.go
+	// Connect and backup
+	if err := backup.Connect(); err != nil {
+		fmt.Printf("%s: Failed to connect: %v\n", hostname, err)
+		return false
+	}
 
-// YAML processing is now handled by the config package
+	err := backup.BackupCommands(commands, excludePatterns, outputDir)
+	backup.Disconnect()
 
-// SSH helper functions are now in ssh.go
+	if err != nil {
+		fmt.Printf("%s: Backup failed: %v\n", hostname, err)
+		return false
+	} else {
+		fmt.Printf("%s: Backup completed\n", hostname)
+		return true
+	}
+}
 
 func main() {
 	var yamlFiles []string
@@ -28,6 +43,7 @@ func main() {
 	var port int
 	var outputDir string
 	var hostFilter []string
+	var parallel int
 
 	var rootCmd = &cobra.Command{
 		Use:     "ipng-router-backup",
@@ -57,17 +73,27 @@ func main() {
 			}
 
 			// Check authentication setup
-			if password == "" && keyFile == "" {
+			hasAuth := 0
-				if os.Getenv("SSH_AUTH_SOCK") != "" {
+			if os.Getenv("SSH_AUTH_SOCK") != "" {
-					fmt.Println("Using SSH agent for authentication")
+				fmt.Println("Using SSH agent for authentication")
-				} else {
+				hasAuth++
-					keyFile = findDefaultSSHKey()
+			}
-					if keyFile == "" {
+			if keyFile == "" {
-						log.Fatal("No SSH key found and no password provided")
+				keyFile = findDefaultSSHKey()
-					} else {
+				if keyFile != "" {
-						fmt.Printf("Using SSH key: %s\n", keyFile)
+					fmt.Printf("Using SSH key: %s\n", keyFile)
-					}
+					hasAuth++
 				}
+			} else {
+				fmt.Printf("Using specified SSH key: %s\n", keyFile)
+				hasAuth++
+			}
+			if password != "" {
+				fmt.Println("Using --password for authentication")
+				hasAuth++
+			}
+			if hasAuth == 0 {
+				log.Fatal("No authentication mechanisms found.")
 			}
 
 			// Process devices
@@ -93,12 +119,41 @@ func main() {
 				}
 			}
 
-			successCount := 0
 			totalCount := len(devicesToProcess)
 
-			for hostname, deviceConfig := range devicesToProcess {
+			// Create channels for work distribution and result collection
-				fmt.Printf("\n%s: Processing device (type: %s)\n", hostname, deviceConfig.Type)
+			type DeviceWork struct {
+				hostname        string
+				deviceConfig    Device
+				commands        []string
+				excludePatterns []string
+			}
 
+			type DeviceResult struct {
+				hostname string
+				success  bool
+			}
+
+			workChan := make(chan DeviceWork, totalCount)
+			resultChan := make(chan DeviceResult, totalCount)
+
+			// Start worker pool
+			var wg sync.WaitGroup
+			for i := 0; i < parallel; i++ {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					for work := range workChan {
+						fmt.Printf("%s: Processing device (type: %s)\n", work.hostname, work.deviceConfig.Type)
+
+						success := processDevice(work.hostname, work.deviceConfig, work.commands, work.excludePatterns, password, keyFile, port, outputDir)
+						resultChan <- DeviceResult{hostname: work.hostname, success: success}
+					}
+				}()
+			}
+
+			// Queue all work
+			for hostname, deviceConfig := range devicesToProcess {
 				user := deviceConfig.User
 				commands := deviceConfig.Commands
 				deviceType := deviceConfig.Type
@@ -122,27 +177,30 @@ func main() {
 					continue
 				}
 
-				// Create backup instance
+				workChan <- DeviceWork{
-				backup := NewRouterBackup(hostname, deviceConfig.Address, user, password, keyFile, port)
+					hostname:        hostname,
-
+					deviceConfig:    deviceConfig,
-				// Connect and backup
+					commands:        commands,
-				if err := backup.Connect(); err != nil {
+					excludePatterns: excludePatterns,
-					fmt.Printf("%s: Failed to connect: %v\n", hostname, err)
-					continue
 				}
+			}
+			close(workChan)
 
-				err = backup.BackupCommands(commands, excludePatterns, outputDir)
+			// Wait for all workers to finish
-				backup.Disconnect()
+			go func() {
+				wg.Wait()
+				close(resultChan)
+			}()
 
-				if err != nil {
+			// Collect results
-					fmt.Printf("%s: Backup failed: %v\n", hostname, err)
+			successCount := 0
-				} else {
+			for result := range resultChan {
-					fmt.Printf("%s: Backup completed\n", hostname)
+				if result.success {
 					successCount++
 				}
 			}
 
-			fmt.Printf("\nOverall summary: %d/%d devices processed successfully\n", successCount, totalCount)
+			fmt.Printf("Overall summary: %d/%d devices processed successfully\n", successCount, totalCount)
 
 			// Set exit code based on results
 			if successCount == 0 {
@@ -160,8 +218,11 @@ func main() {
 	rootCmd.Flags().IntVar(&port, "port", 22, "SSH port")
 	rootCmd.Flags().StringVar(&outputDir, "output-dir", "/tmp", "Output directory for command output files")
 	rootCmd.Flags().StringSliceVar(&hostFilter, "host", []string{}, "Specific host(s) to process (can be repeated, processes all if not specified)")
+	rootCmd.Flags().IntVar(&parallel, "parallel", 10, "Maximum number of devices to process in parallel")
 
-	rootCmd.MarkFlagRequired("yaml")
+	if err := rootCmd.MarkFlagRequired("yaml"); err != nil {
+		log.Fatal(err)
+	}
 
 	if err := rootCmd.Execute(); err != nil {
 		log.Fatal(err)

src/ssh.go

@@ -4,7 +4,6 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
 	"net"
 	"os"
 	"path/filepath"
@@ -104,11 +103,6 @@ func (rb *RouterBackup) Connect() error {
 		config.KeyExchanges = finalAlgorithms
 	}
 
-	// Note: Cipher overrides disabled - Go SSH library defaults work better
-	// if ciphers := ssh_config.Get(rb.hostname, "Ciphers"); ciphers != "" {
-	//	 config.Ciphers = ...
-	// }
-
 	if macs := ssh_config.Get(rb.hostname, "MACs"); macs != "" {
 		macList := strings.Split(macs, ",")
 		for i, mac := range macList {
@@ -127,15 +121,19 @@ func (rb *RouterBackup) Connect() error {
 		config.HostKeyAlgorithms = finalAlgorithms
 	}
 
-	// Try SSH agent first if available
+	// If explicit key file is provided, prioritize it over SSH agent
+	var keyFileAuth ssh.AuthMethod
+	var agentAuth ssh.AuthMethod
+
+	// Try SSH agent if available (but don't add to config.Auth yet)
 	if sshAuthSock := os.Getenv("SSH_AUTH_SOCK"); sshAuthSock != "" {
 		if conn, err := net.Dial("unix", sshAuthSock); err == nil {
 			agentClient := agent.NewClient(conn)
-			config.Auth = []ssh.AuthMethod{ssh.PublicKeysCallback(agentClient.Signers)}
+			agentAuth = ssh.PublicKeysCallback(agentClient.Signers)
 		}
 	}
 
-	// If SSH agent didn't work, try key file
+	// Try key file
 	if keyFile != "" {
 		// Expand ~ in keyFile path
 		if strings.HasPrefix(keyFile, "~/") {
@@ -145,17 +143,27 @@ func (rb *RouterBackup) Connect() error {
 			}
 		}
 
-		key, err := ioutil.ReadFile(keyFile)
+		key, err := os.ReadFile(keyFile)
 		if err == nil {
 			signer, err := ssh.ParsePrivateKey(key)
 			if err != nil {
-				fmt.Errorf("unable to parse private key: %v", err)
+				fmt.Printf("%s: Unable to parse private key: %v\n", rb.hostname, err)
 			} else {
-				config.Auth = append(config.Auth, ssh.PublicKeys(signer))
+				keyFileAuth = ssh.PublicKeys(signer)
 			}
 		}
 	}
 
+	// Prioritize auth methods: explicit key file first, then SSH agent
+	if keyFileAuth != nil {
+		config.Auth = []ssh.AuthMethod{keyFileAuth}
+		if agentAuth != nil {
+			config.Auth = append(config.Auth, agentAuth)
+		}
+	} else if agentAuth != nil {
+		config.Auth = []ssh.AuthMethod{agentAuth}
+	}
+
 	// Fall back to password if available
 	if rb.password != "" {
 		config.Auth = append(config.Auth, ssh.Password(rb.password))
@@ -268,7 +276,10 @@ func (rb *RouterBackup) BackupCommands(commands []string, excludePatterns []stri
 
 		fmt.Fprintf(file, "## COMMAND: %s\n", command)
 		filteredOutput := filterOutput(output, excludePatterns)
-		file.WriteString(filteredOutput)
+		if _, err := file.WriteString(filteredOutput); err != nil {
+			fmt.Printf("%s: Failed to write output: %v\n", rb.hostname, err)
+			hasErrors = true
+		}
 		file.Close()
 
 		successCount++