Cut v1.0.2

.gitignore

@@ -1,4 +1,5 @@
 ipng-router-backup
+*.yaml
 
 # Debian packaging artifacts
 debian/.debhelper/

README.md

@@ -6,6 +6,7 @@ SSH-based network device configuration backup tool with support for multiple dev
 
 - **Multi-device backup**: Configure multiple routers in YAML
 - **Device type templates**: Reusable command sets per device type
+- **Configuration includes**: Split large configs with `!include` directives
 - **Flexible authentication**: SSH agent, key files, or password
 - **Selective execution**: Backup specific devices with `--host` flags
 - **Professional CLI**: Standard flags, version info, and help
@@ -24,24 +25,30 @@ make build
 
 ### Basic Usage
 
-1. **Create configuration file** (`config.yaml`):
+1. **Create configuration files**:
 
-```yaml
+   **Main config** (`config.yaml`):
-types:
+   ```yaml
-  srlinux:
+   !include device-types.yaml
-    commands:
-      - show version
-      - show platform linecard
-      - info flat from running
 
-devices:
+   devices:
-  asw100:
+     asw100:
-    user: admin
+       user: admin
-    type: srlinux
+       type: srlinux
-  asw120:
+     asw120:
-    user: admin  
+       user: admin
-    type: srlinux
+       type: srlinux
-```
+   ```
+
+   **Device types** (`device-types.yaml`):
+   ```yaml
+   types:
+     srlinux:
+       commands:
+         - show version
+         - show platform linecard
+         - info flat from running
+   ```
 
 2. **Run backup**:
 

config.yaml

@@ -1,46 +0,0 @@
-types:
-  srlinux:
-    commands:
-      - show version
-      - show platform linecard
-      - show platform fan-tray
-      - show platform power-supply
-      - info flat from running
-  eos:
-    commands:
-      - show version
-      - show inventory
-      - show env power
-      - show running-config
-  centec:
-    commands:
-      - show version | exc uptime
-      - show boot images
-      - show transceiver
-      - show running-config
-
-devices:
-  csw150:
-    user: pim
-    type: centec
-  csw151:
-    user: pim
-    type: centec
-  asw100:
-    user: pim
-    type: srlinux
-  asw120:
-    user: pim
-    type: srlinux
-  asw121:
-    user: pim
-    type: srlinux
-  asw110:
-    user: pim
-    type: eos
-  asw111:
-    user: pim
-    type: eos
-  asw112:
-    user: pim
-    type: eos

debian/changelog

@@ -1,3 +1,10 @@
+ipng-router-backup (1.0.2) stable; urgency=low
+
+  * Add YAML !include directive support for configuration files
+  * Add docs/device-types.yaml example file to package
+
+ -- Pim van Pelt <pim@ipng.ch>  Sun, 06 Jul 2025 12:25:00 +0100
+
 ipng-router-backup (1.0.1) stable; urgency=low
 
   * Add version information to help output

debian/rules

@@ -11,7 +11,8 @@ override_dh_auto_install:
 	mkdir -p debian/ipng-router-backup/etc/ipng-router-backup
 	mkdir -p debian/ipng-router-backup/usr/share/man/man1
 	cp ipng-router-backup debian/ipng-router-backup/usr/bin/
-	cp config.yaml debian/ipng-router-backup/etc/ipng-router-backup/config.yaml.example
+	cp docs/config.yaml.example debian/ipng-router-backup/etc/ipng-router-backup/config.yaml.example
+	cp docs/device-types.yaml debian/ipng-router-backup/etc/ipng-router-backup/device-types.yaml
 	cp docs/router_backup.1 debian/ipng-router-backup/usr/share/man/man1/ipng-router-backup.1
 	gzip debian/ipng-router-backup/usr/share/man/man1/ipng-router-backup.1
 

docs/DETAILS.md

@@ -8,6 +8,7 @@ IPng Networks Router Backup is a SSH-based network device configuration backup t
 
 - **Multi-device support**: Backup multiple routers in a single run
 - **Device type templates**: Define command sets per device type
+- **Configuration includes**: Split large configurations with `!include` directives
 - **Flexible authentication**: SSH agent, key files, or password authentication
 - **Selective execution**: Target specific devices with `--host` flags
 - **Automatic file organization**: Output files named by hostname
@@ -16,10 +17,35 @@ IPng Networks Router Backup is a SSH-based network device configuration backup t
 
 ## Configuration File Format
 
-The tool uses a YAML configuration file with two main sections: `types` and `devices`.
+The tool uses a YAML configuration file with two main sections: `types` and `devices`. The configuration supports `!include` directives for organizing large configurations across multiple files.
 
 ### Complete Example
 
+**Main configuration** (`config.yaml`):
+```yaml
+!include device-types.yaml
+
+devices:
+  asw100:
+    user: admin
+    type: srlinux
+
+  asw120:
+    user: netops
+    type: srlinux
+
+  core-01:
+    user: admin
+    type: eos
+
+  edge-router:
+    user: operator
+    commands:
+      - show version
+      - show ip route summary
+```
+
+**Device types file** (`device-types.yaml`):
 ```yaml
 types:
   srlinux:
@@ -29,39 +55,20 @@ types:
       - show platform fan-tray
       - show platform power-supply
       - info flat from running
-  
+
   eos:
     commands:
       - show version
       - show inventory
       - show env power
       - show running-config
-  
+
   centec:
     commands:
       - show version | exc uptime
       - show boot images
       - show transceiver
       - show running-config
-
-devices:
-  asw100:
-    user: admin
-    type: srlinux
-  
-  asw120:
-    user: netops
-    type: srlinux
-  
-  core-01:
-    user: admin
-    type: eos
-  
-  edge-router:
-    user: operator
-    commands:
-      - show version
-      - show ip route summary
 ```
 
 ### Configuration Fields
@@ -89,6 +96,61 @@ devices:
 - Type references must exist in the `types` section
 - Commands can be specified either via type reference or directly per device
 
+### Include Directive Support
+
+The configuration supports `!include` directives for splitting large configurations into multiple files:
+
+```yaml
+# Main config.yaml
+!include device-types.yaml
+
+devices:
+  production-device:
+    user: admin
+    type: srlinux
+```
+
+**Include Features:**
+- **One level deep**: Included files cannot contain their own `!include` directives
+- **Relative paths**: Paths are relative to the including file's directory
+- **Absolute paths**: Fully qualified paths are supported
+- **Quoted paths**: Use quotes for paths containing spaces: `!include "file with spaces.yaml"`
+- **Proper indentation**: Included content maintains correct YAML indentation
+
+**Example file structure:**
+```
+/etc/ipng-router-backup/
+├── config.yaml              # Main configuration with !include
+├── device-types.yaml        # Device type definitions
+└── devices/
+    ├── production.yaml      # Production device definitions
+    └── lab.yaml            # Lab device definitions
+```
+
+**Usage patterns:**
+
+1. **Include device types at top level:**
+   ```yaml
+   !include device-types.yaml
+
+   devices:
+     # device definitions here
+   ```
+
+2. **Include under specific sections:**
+   ```yaml
+   types:
+     !include types/network-devices.yaml
+
+   devices:
+     !include devices/production.yaml
+   ```
+
+3. **Include files with spaces:**
+   ```yaml
+   !include "device types/lab environment.yaml"
+   ```
+
 ## Command Line Flags
 
 ### Required Flags
@@ -109,7 +171,7 @@ devices:
 
 ```bash
 # Basic usage - all devices
-ipng-router-backup --config /etc/network-backup/config.yaml
+ipng-router-backup --config /etc/ipng-router-backup/config.yaml
 
 # Custom output directory
 ipng-router-backup --config config.yaml --output-dir /backup/network
@@ -163,7 +225,7 @@ ipng-router-backup --config config.yaml --key-file ~/.ssh/network_key
 
 # Tool automatically checks these default locations:
 # ~/.ssh/id_rsa
-# ~/.ssh/id_ed25519  
+# ~/.ssh/id_ed25519
 # ~/.ssh/id_ecdsa
 ```
 
@@ -255,7 +317,7 @@ BACKUP_DIR="/backup/network/$(date +%Y%m%d)"
 mkdir -p "$BACKUP_DIR"
 
 ipng-router-backup \
-  --config /etc/network-backup/config.yaml \
+  --config /etc/ipng-router-backup/config.yaml \
   --output-dir "$BACKUP_DIR"
 
 # Kill SSH agent
@@ -299,6 +361,55 @@ ipng-router-backup \
 
 ## Advanced Usage
 
+### Configuration Organization with Includes
+
+For large deployments, organize configurations using `!include` directives:
+
+**Environment-based structure:**
+```bash
+network-backup/
+├── config.yaml                    # Main config
+├── types/
+│   ├── device-types.yaml         # All device types
+│   └── vendor-specific.yaml      # Vendor-specific commands
+├── environments/
+│   ├── production.yaml           # Production devices
+│   ├── staging.yaml              # Staging devices
+│   └── lab.yaml                  # Lab devices
+└── sites/
+    ├── datacenter-east.yaml      # East datacenter devices
+    └── datacenter-west.yaml      # West datacenter devices
+```
+
+**Main configuration** (`config.yaml`):
+```yaml
+!include types/device-types.yaml
+
+devices:
+  # Production environment
+  !include environments/production.yaml
+
+  # Lab environment
+  !include environments/lab.yaml
+```
+
+**Production devices** (`environments/production.yaml`):
+```yaml
+# Production SR Linux switches
+prod-asw100:
+  user: netops
+  type: srlinux
+
+prod-asw120:
+  user: netops
+  type: srlinux
+
+# Production EOS devices
+prod-core-01:
+  user: netops
+  type: eos
+```
+
 ### Integration with Git
 
 ```bash
@@ -324,9 +435,9 @@ types:
   production-srlinux:
     commands:
       - show version
-      - show system information  
+      - show system information
       - info flat from running
-      
+
   lab-srlinux:
     commands:
       - show version
@@ -336,7 +447,7 @@ devices:
   prod-asw100:
     user: readonly
     type: production-srlinux
-    
+
   lab-asw100:
     user: admin
     type: lab-srlinux

docs/config.yaml.example

@@ -0,0 +1,98 @@
+# IPng Networks Router Backup Configuration Example
+# Copyright 2025, IPng Networks GmbH, Pim van Pelt <pim@ipng.ch>
+#
+# This file demonstrates how to configure the ipng-router-backup tool.
+# Copy this file to a location of your choice and modify for your environment.
+#
+# Usage: ipng-router-backup --config /path/to/your/config.yaml
+#
+# YAML !include Support:
+# You can split large configurations into multiple files using !include directives.
+# Examples:
+#   !include device-types.yaml
+#   !include devices/production.yaml
+#   !include "devices/lab environment.yaml"  # Use quotes for paths with spaces
+
+# Include device types from separate file
+!include device-types.yaml
+
+# Devices Section
+# Define individual network devices to backup
+devices:
+  # Core switches (SR Linux)
+  asw100:
+    user: admin                         # SSH username
+    type: srlinux                       # Reference to type above
+
+  asw120:
+    user: netops                        # Different user per device if needed
+    type: srlinux
+
+  asw121:
+    user: admin
+    type: srlinux
+
+  # Distribution switches (Centec)
+  csw150:
+    user: admin
+    type: centec
+
+  csw151:
+    user: admin
+    type: centec
+
+  # Edge routers (Arista EOS)
+  edge-01:
+    user: automation
+    type: eos
+
+  edge-02:
+    user: automation
+    type: eos
+
+  # Special case: Device with custom commands (overrides type)
+  legacy-router:
+    user: admin
+    commands:
+      - show version
+      - show running-config
+      - show ip route summary
+      # Custom commands specific to this device only
+
+  # Example using IP address instead of hostname
+  192.168.1.100:
+    user: operator
+    type: cisco-ios
+
+# Configuration Tips:
+#
+# 1. Authentication Priority (automatic):
+#    - SSH Agent (if SSH_AUTH_SOCK environment variable is set)
+#    - SSH Key file (--key-file flag or default locations)
+#    - Password (--password flag or interactive prompt)
+#
+# 2. Running the backup:
+#    # Backup all devices
+#    ipng-router-backup --config /etc/ipng-router-backup/config.yaml
+#
+#    # Backup specific devices only
+#    ipng-router-backup --config config.yaml --host asw100 --host edge-01
+#
+#    # Custom output directory
+#    ipng-router-backup --config config.yaml --output-dir /backup/$(date +%Y%m%d)
+#
+# 3. Output files:
+#    - Named after device hostname (e.g., 'asw100', 'edge-01')
+#    - Each command output prefixed with "## COMMAND: <command>"
+#    - Files are recreated on each run (not appended)
+#
+# 4. Security considerations:
+#    - Use SSH keys instead of passwords when possible
+#    - Consider using SSH agent for additional security
+#    - Restrict SSH access to backup user accounts
+#    - Store configuration files with appropriate permissions (640 recommended)
+#
+# 5. Error handling:
+#    - If a device is unreachable, the tool continues with other devices
+#    - Check tool output for connection or authentication failures
+#    - Use --host flag to test individual devices

src/main.go

@@ -9,6 +9,8 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"regexp"
+	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -17,7 +19,7 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
-const Version = "1.0.1"
+const Version = "1.0.2"
 
 // Config structures
 type Config struct {
@@ -180,15 +182,15 @@ func (rb *RouterBackup) Disconnect() {
 	}
 }
 
-// loadConfig loads the YAML configuration file
+// loadConfig loads the YAML configuration file with !include support
 func loadConfig(configPath string) (*Config, error) {
-	data, err := ioutil.ReadFile(configPath)
+	processedYAML, err := processIncludes(configPath)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read config file %s: %v", configPath, err)
+		return nil, fmt.Errorf("failed to process includes: %v", err)
 	}
 
 	var config Config
-	err = yaml.Unmarshal(data, &config)
+	err = yaml.Unmarshal([]byte(processedYAML), &config)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse YAML: %v", err)
 	}
@@ -196,6 +198,75 @@ func loadConfig(configPath string) (*Config, error) {
 	return &config, nil
 }
 
+// processIncludes processes YAML files with !include directives (one level deep)
+func processIncludes(filePath string) (string, error) {
+	// Read the file
+	data, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return "", fmt.Errorf("failed to read file %s: %v", filePath, err)
+	}
+
+	content := string(data)
+
+	// Process !include directives
+	// Match patterns like: !include path/to/file.yaml (excluding commented lines)
+	includeRegex := regexp.MustCompile(`(?m)^(\s*)!include\s+(.+)$`)
+
+	baseDir := filepath.Dir(filePath)
+
+	// Process includes line by line to avoid conflicts
+	lines := strings.Split(content, "\n")
+	var resultLines []string
+
+	for _, line := range lines {
+		// Check if this line matches our include pattern
+		if match := includeRegex.FindStringSubmatch(line); match != nil {
+			leadingWhitespace := match[1]
+			includePath := strings.TrimSpace(match[2])
+
+			// Skip commented lines
+			if strings.Contains(strings.TrimSpace(line), "#") && strings.Index(strings.TrimSpace(line), "#") < strings.Index(strings.TrimSpace(line), "!include") {
+				resultLines = append(resultLines, line)
+				continue
+			}
+
+			// Remove quotes if present
+			includePath = strings.Trim(includePath, "\"'")
+
+			// Make path relative to current config file
+			if !filepath.IsAbs(includePath) {
+				includePath = filepath.Join(baseDir, includePath)
+			}
+
+			// Read the included file
+			includedData, err := ioutil.ReadFile(includePath)
+			if err != nil {
+				return "", fmt.Errorf("failed to read include file %s: %v", includePath, err)
+			}
+
+			// Use the captured leading whitespace as indentation prefix
+			indentPrefix := leadingWhitespace
+
+			// Indent each line of included content to match the !include line's indentation
+			includedLines := strings.Split(string(includedData), "\n")
+			for _, includeLine := range includedLines {
+				if strings.TrimSpace(includeLine) == "" {
+					resultLines = append(resultLines, "")
+				} else {
+					resultLines = append(resultLines, indentPrefix+includeLine)
+				}
+			}
+		} else {
+			// Regular line, just copy it
+			resultLines = append(resultLines, line)
+		}
+	}
+
+	content = strings.Join(resultLines, "\n")
+
+	return content, nil
+}
+
 // findDefaultSSHKey looks for default SSH keys
 func findDefaultSSHKey() string {
 	homeDir, err := os.UserHomeDir()

src/main_test.go

@@ -5,6 +5,7 @@ package main
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 )
 
@@ -71,15 +72,25 @@ func TestFindDefaultSSHKeyNotFound(t *testing.T) {
 }
 
 func TestLoadConfig(t *testing.T) {
-	// Create a temporary config file
+	// Create a temporary directory and files
 	tempDir := t.TempDir()
-	configPath := filepath.Join(tempDir, "test-config.yaml")
 
+	// Create device-types.yaml file
+	deviceTypesPath := filepath.Join(tempDir, "device-types.yaml")
+	deviceTypesContent := `test-type:
+  commands:
+    - show version
+    - show status`
+
+	err := os.WriteFile(deviceTypesPath, []byte(deviceTypesContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create device-types file: %v", err)
+	}
+
+	// Create main config file with !include
+	configPath := filepath.Join(tempDir, "test-config.yaml")
 	configContent := `types:
-  test-type:
+  !include device-types.yaml
-    commands:
-      - show version
-      - show status
 
 devices:
   test-device:
@@ -91,7 +102,7 @@ devices:
       - direct command
 `
 
-	err := os.WriteFile(configPath, []byte(configContent), 0644)
+	err = os.WriteFile(configPath, []byte(configContent), 0644)
 	if err != nil {
 		t.Fatalf("Failed to create test config file: %v", err)
 	}
@@ -285,3 +296,179 @@ func TestRouterBackupCreation(t *testing.T) {
 		})
 	}
 }
+
+// Test !include functionality
+func TestProcessIncludes(t *testing.T) {
+	tempDir := t.TempDir()
+
+	// Create included file
+	includedPath := filepath.Join(tempDir, "included.yaml")
+	includedContent := `test-type:
+  commands:
+    - show version
+    - show status`
+
+	err := os.WriteFile(includedPath, []byte(includedContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create included file: %v", err)
+	}
+
+	// Create main file with !include
+	mainPath := filepath.Join(tempDir, "main.yaml")
+	mainContent := `types:
+  !include included.yaml
+devices:
+  test-device:
+    user: testuser
+    type: test-type`
+
+	err = os.WriteFile(mainPath, []byte(mainContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create main file: %v", err)
+	}
+
+	// Process includes
+	result, err := processIncludes(mainPath)
+	if err != nil {
+		t.Fatalf("Failed to process includes: %v", err)
+	}
+
+	// Check that include was processed
+	if !strings.Contains(result, "show version") {
+		t.Error("Expected included content to be present in result")
+	}
+	if !strings.Contains(result, "show status") {
+		t.Error("Expected included content to be present in result")
+	}
+	if strings.Contains(result, "!include") {
+		t.Error("Expected !include directive to be replaced")
+	}
+}
+
+func TestProcessIncludesWithQuotes(t *testing.T) {
+	tempDir := t.TempDir()
+
+	// Create included file with spaces in name
+	includedPath := filepath.Join(tempDir, "file with spaces.yaml")
+	includedContent := `production-srlinux:
+  commands:
+    - show version`
+
+	err := os.WriteFile(includedPath, []byte(includedContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create included file: %v", err)
+	}
+
+	// Create main file with quoted !include
+	mainPath := filepath.Join(tempDir, "main.yaml")
+	mainContent := `types:
+  !include "file with spaces.yaml"`
+
+	err = os.WriteFile(mainPath, []byte(mainContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create main file: %v", err)
+	}
+
+	// Process includes
+	result, err := processIncludes(mainPath)
+	if err != nil {
+		t.Fatalf("Failed to process includes: %v", err)
+	}
+
+	// Check that include was processed
+	if !strings.Contains(result, "production-srlinux") {
+		t.Error("Expected included content to be present in result")
+	}
+}
+
+func TestProcessIncludesNonexistentFile(t *testing.T) {
+	tempDir := t.TempDir()
+
+	// Create main file with include to nonexistent file
+	mainPath := filepath.Join(tempDir, "main.yaml")
+	mainContent := `types:
+  !include nonexistent.yaml`
+
+	err := os.WriteFile(mainPath, []byte(mainContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create main file: %v", err)
+	}
+
+	// Process includes should fail
+	_, err = processIncludes(mainPath)
+	if err == nil {
+		t.Error("Expected error for nonexistent include file")
+	}
+}
+
+func TestLoadConfigWithIncludes(t *testing.T) {
+	tempDir := t.TempDir()
+
+	// Create device types file
+	typesPath := filepath.Join(tempDir, "types.yaml")
+	typesContent := `srlinux:
+  commands:
+    - show version
+    - show platform linecard
+eos:
+  commands:
+    - show version
+    - show inventory`
+
+	err := os.WriteFile(typesPath, []byte(typesContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create types file: %v", err)
+	}
+
+	// Create main config file with includes
+	mainPath := filepath.Join(tempDir, "config.yaml")
+	mainContent := `types:
+  !include types.yaml
+devices:
+  asw100:
+    user: admin
+    type: srlinux
+  edge-01:
+    user: operator
+    type: eos`
+
+	err = os.WriteFile(mainPath, []byte(mainContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create main config file: %v", err)
+	}
+
+	// Load configuration
+	config, err := loadConfig(mainPath)
+	if err != nil {
+		t.Fatalf("Failed to load config with includes: %v", err)
+	}
+
+	// Verify types were loaded correctly
+	if len(config.Types) != 2 {
+		t.Errorf("Expected 2 types, got %d", len(config.Types))
+	}
+
+	srlinuxType, exists := config.Types["srlinux"]
+	if !exists {
+		t.Error("Expected 'srlinux' type to exist")
+	}
+	if len(srlinuxType.Commands) != 2 {
+		t.Errorf("Expected 2 commands for srlinux type, got %d", len(srlinuxType.Commands))
+	}
+
+	// Verify devices were loaded correctly
+	if len(config.Devices) != 2 {
+		t.Errorf("Expected 2 devices, got %d", len(config.Devices))
+	}
+
+	asw100, exists := config.Devices["asw100"]
+	if !exists {
+		t.Error("Expected 'asw100' device to exist")
+	}
+	if asw100.User != "admin" {
+		t.Errorf("Expected user 'admin', got '%s'", asw100.User)
+	}
+	if asw100.Type != "srlinux" {
+		t.Errorf("Expected type 'srlinux', got '%s'", asw100.Type)
+	}
+}