add yaml include feature

.gitignore

@@ -1,5 +1,5 @@
 ipng-router-backup
-config.yaml
+*.yaml
 
 # Debian packaging artifacts
 debian/.debhelper/

README.md

@@ -6,6 +6,7 @@ SSH-based network device configuration backup tool with support for multiple dev
 
 - **Multi-device backup**: Configure multiple routers in YAML
 - **Device type templates**: Reusable command sets per device type
+- **Configuration includes**: Split large configs with `!include` directives
 - **Flexible authentication**: SSH agent, key files, or password
 - **Selective execution**: Backup specific devices with `--host` flags
 - **Professional CLI**: Standard flags, version info, and help
@@ -24,15 +25,11 @@ make build
 
 ### Basic Usage
 
-1. **Create configuration file** (`config.yaml`):
+1. **Create configuration files**:
 
+   **Main config** (`config.yaml`):
    ```yaml
-types:
+   !include device-types.yaml
-  srlinux:
-    commands:
-      - show version
-      - show platform linecard
-      - info flat from running
 
    devices:
      asw100:
@@ -43,6 +40,16 @@ devices:
        type: srlinux
    ```
 
+   **Device types** (`device-types.yaml`):
+   ```yaml
+   types:
+     srlinux:
+       commands:
+         - show version
+         - show platform linecard
+         - info flat from running
+   ```
+
 2. **Run backup**:
 
 ```bash

debian/rules

@@ -12,6 +12,7 @@ override_dh_auto_install:
 	mkdir -p debian/ipng-router-backup/usr/share/man/man1
 	cp ipng-router-backup debian/ipng-router-backup/usr/bin/
 	cp docs/config.yaml.example debian/ipng-router-backup/etc/ipng-router-backup/config.yaml.example
+	cp docs/device-types.yaml debian/ipng-router-backup/etc/ipng-router-backup/device-types.yaml
 	cp docs/router_backup.1 debian/ipng-router-backup/usr/share/man/man1/ipng-router-backup.1
 	gzip debian/ipng-router-backup/usr/share/man/man1/ipng-router-backup.1
 

docs/DETAILS.md

@@ -8,6 +8,7 @@ IPng Networks Router Backup is a SSH-based network device configuration backup t
 
 - **Multi-device support**: Backup multiple routers in a single run
 - **Device type templates**: Define command sets per device type
+- **Configuration includes**: Split large configurations with `!include` directives
 - **Flexible authentication**: SSH agent, key files, or password authentication
 - **Selective execution**: Target specific devices with `--host` flags
 - **Automatic file organization**: Output files named by hostname
@@ -16,10 +17,35 @@ IPng Networks Router Backup is a SSH-based network device configuration backup t
 
 ## Configuration File Format
 
-The tool uses a YAML configuration file with two main sections: `types` and `devices`.
+The tool uses a YAML configuration file with two main sections: `types` and `devices`. The configuration supports `!include` directives for organizing large configurations across multiple files.
 
 ### Complete Example
 
+**Main configuration** (`config.yaml`):
+```yaml
+!include device-types.yaml
+
+devices:
+  asw100:
+    user: admin
+    type: srlinux
+
+  asw120:
+    user: netops
+    type: srlinux
+
+  core-01:
+    user: admin
+    type: eos
+
+  edge-router:
+    user: operator
+    commands:
+      - show version
+      - show ip route summary
+```
+
+**Device types file** (`device-types.yaml`):
 ```yaml
 types:
   srlinux:
@@ -43,25 +69,6 @@ types:
       - show boot images
       - show transceiver
       - show running-config
-
-devices:
-  asw100:
-    user: admin
-    type: srlinux
-  
-  asw120:
-    user: netops
-    type: srlinux
-  
-  core-01:
-    user: admin
-    type: eos
-  
-  edge-router:
-    user: operator
-    commands:
-      - show version
-      - show ip route summary
 ```
 
 ### Configuration Fields
@@ -89,6 +96,61 @@ devices:
 - Type references must exist in the `types` section
 - Commands can be specified either via type reference or directly per device
 
+### Include Directive Support
+
+The configuration supports `!include` directives for splitting large configurations into multiple files:
+
+```yaml
+# Main config.yaml
+!include device-types.yaml
+
+devices:
+  production-device:
+    user: admin
+    type: srlinux
+```
+
+**Include Features:**
+- **One level deep**: Included files cannot contain their own `!include` directives
+- **Relative paths**: Paths are relative to the including file's directory
+- **Absolute paths**: Fully qualified paths are supported
+- **Quoted paths**: Use quotes for paths containing spaces: `!include "file with spaces.yaml"`
+- **Proper indentation**: Included content maintains correct YAML indentation
+
+**Example file structure:**
+```
+/etc/ipng-router-backup/
+├── config.yaml              # Main configuration with !include
+├── device-types.yaml        # Device type definitions
+└── devices/
+    ├── production.yaml      # Production device definitions
+    └── lab.yaml            # Lab device definitions
+```
+
+**Usage patterns:**
+
+1. **Include device types at top level:**
+   ```yaml
+   !include device-types.yaml
+
+   devices:
+     # device definitions here
+   ```
+
+2. **Include under specific sections:**
+   ```yaml
+   types:
+     !include types/network-devices.yaml
+
+   devices:
+     !include devices/production.yaml
+   ```
+
+3. **Include files with spaces:**
+   ```yaml
+   !include "device types/lab environment.yaml"
+   ```
+
 ## Command Line Flags
 
 ### Required Flags
@@ -109,7 +171,7 @@ devices:
 
 ```bash
 # Basic usage - all devices
-ipng-router-backup --config /etc/network-backup/config.yaml
+ipng-router-backup --config /etc/ipng-router-backup/config.yaml
 
 # Custom output directory
 ipng-router-backup --config config.yaml --output-dir /backup/network
@@ -255,7 +317,7 @@ BACKUP_DIR="/backup/network/$(date +%Y%m%d)"
 mkdir -p "$BACKUP_DIR"
 
 ipng-router-backup \
-  --config /etc/network-backup/config.yaml \
+  --config /etc/ipng-router-backup/config.yaml \
   --output-dir "$BACKUP_DIR"
 
 # Kill SSH agent
@@ -299,6 +361,55 @@ ipng-router-backup \
 
 ## Advanced Usage
 
+### Configuration Organization with Includes
+
+For large deployments, organize configurations using `!include` directives:
+
+**Environment-based structure:**
+```bash
+network-backup/
+├── config.yaml                    # Main config
+├── types/
+│   ├── device-types.yaml         # All device types
+│   └── vendor-specific.yaml      # Vendor-specific commands
+├── environments/
+│   ├── production.yaml           # Production devices
+│   ├── staging.yaml              # Staging devices
+│   └── lab.yaml                  # Lab devices
+└── sites/
+    ├── datacenter-east.yaml      # East datacenter devices
+    └── datacenter-west.yaml      # West datacenter devices
+```
+
+**Main configuration** (`config.yaml`):
+```yaml
+!include types/device-types.yaml
+
+devices:
+  # Production environment
+  !include environments/production.yaml
+
+  # Lab environment
+  !include environments/lab.yaml
+```
+
+**Production devices** (`environments/production.yaml`):
+```yaml
+# Production SR Linux switches
+prod-asw100:
+  user: netops
+  type: srlinux
+
+prod-asw120:
+  user: netops
+  type: srlinux
+
+# Production EOS devices
+prod-core-01:
+  user: netops
+  type: eos
+```
+
 ### Integration with Git
 
 ```bash

docs/config.yaml.example

@@ -5,51 +5,16 @@
 # Copy this file to a location of your choice and modify for your environment.
 #
 # Usage: ipng-router-backup --config /path/to/your/config.yaml
+#
+# YAML !include Support:
+# You can split large configurations into multiple files using !include directives.
+# Examples:
+#   !include device-types.yaml
+#   !include devices/production.yaml
+#   !include "devices/lab environment.yaml"  # Use quotes for paths with spaces
 
-# Device Types Section
+# Include device types from separate file
-# Define reusable command sets for different types of network equipment
+!include device-types.yaml
-types:
-  # Nokia SR Linux devices
-  srlinux:
-    commands:
-      - show version                    # System version and build info
-      - show platform linecard         # Line card information
-      - show platform fan-tray         # Fan status and health
-      - show platform power-supply     # Power supply status
-      - info flat from running         # Full running configuration
-
-  # Arista EOS devices  
-  eos:
-    commands:
-      - show version                    # System version information
-      - show inventory                  # Hardware inventory
-      - show env power                  # Power supply status
-      - show running-config             # Complete running configuration
-
-  # Centec switches
-  centec:
-    commands:
-      - show version | exc uptime       # Version info without uptime line
-      - show boot images                # Boot image information
-      - show transceiver                # SFP/transceiver status
-      - show running-config             # Running configuration
-
-  # Cisco IOS/IOS-XE devices
-  cisco-ios:
-    commands:
-      - show version                    # IOS version and hardware info
-      - show inventory                  # Hardware inventory details
-      - show running-config             # Complete configuration
-      - show ip interface brief         # Interface IP summary
-      - show cdp neighbors              # CDP neighbor information
-
-  # Juniper devices
-  junos:
-    commands:
-      - show version                    # Software and hardware version
-      - show chassis hardware           # Chassis hardware details
-      - show configuration | display set # Configuration in set format
-      - show interfaces terse           # Interface status summary
 
 # Devices Section
 # Define individual network devices to backup

src/main.go

@@ -9,6 +9,8 @@ import (
 	"net"
 	"os"
 	"path/filepath"
+	"regexp"
+	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -180,15 +182,15 @@ func (rb *RouterBackup) Disconnect() {
 	}
 }
 
-// loadConfig loads the YAML configuration file
+// loadConfig loads the YAML configuration file with !include support
 func loadConfig(configPath string) (*Config, error) {
-	data, err := ioutil.ReadFile(configPath)
+	processedYAML, err := processIncludes(configPath)
 	if err != nil {
-		return nil, fmt.Errorf("failed to read config file %s: %v", configPath, err)
+		return nil, fmt.Errorf("failed to process includes: %v", err)
 	}
 
 	var config Config
-	err = yaml.Unmarshal(data, &config)
+	err = yaml.Unmarshal([]byte(processedYAML), &config)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse YAML: %v", err)
 	}
@@ -196,6 +198,75 @@ func loadConfig(configPath string) (*Config, error) {
 	return &config, nil
 }
 
+// processIncludes processes YAML files with !include directives (one level deep)
+func processIncludes(filePath string) (string, error) {
+	// Read the file
+	data, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return "", fmt.Errorf("failed to read file %s: %v", filePath, err)
+	}
+
+	content := string(data)
+
+	// Process !include directives
+	// Match patterns like: !include path/to/file.yaml (excluding commented lines)
+	includeRegex := regexp.MustCompile(`(?m)^(\s*)!include\s+(.+)$`)
+
+	baseDir := filepath.Dir(filePath)
+
+	// Process includes line by line to avoid conflicts
+	lines := strings.Split(content, "\n")
+	var resultLines []string
+
+	for _, line := range lines {
+		// Check if this line matches our include pattern
+		if match := includeRegex.FindStringSubmatch(line); match != nil {
+			leadingWhitespace := match[1]
+			includePath := strings.TrimSpace(match[2])
+
+			// Skip commented lines
+			if strings.Contains(strings.TrimSpace(line), "#") && strings.Index(strings.TrimSpace(line), "#") < strings.Index(strings.TrimSpace(line), "!include") {
+				resultLines = append(resultLines, line)
+				continue
+			}
+
+			// Remove quotes if present
+			includePath = strings.Trim(includePath, "\"'")
+
+			// Make path relative to current config file
+			if !filepath.IsAbs(includePath) {
+				includePath = filepath.Join(baseDir, includePath)
+			}
+
+			// Read the included file
+			includedData, err := ioutil.ReadFile(includePath)
+			if err != nil {
+				return "", fmt.Errorf("failed to read include file %s: %v", includePath, err)
+			}
+
+			// Use the captured leading whitespace as indentation prefix
+			indentPrefix := leadingWhitespace
+
+			// Indent each line of included content to match the !include line's indentation
+			includedLines := strings.Split(string(includedData), "\n")
+			for _, includeLine := range includedLines {
+				if strings.TrimSpace(includeLine) == "" {
+					resultLines = append(resultLines, "")
+				} else {
+					resultLines = append(resultLines, indentPrefix+includeLine)
+				}
+			}
+		} else {
+			// Regular line, just copy it
+			resultLines = append(resultLines, line)
+		}
+	}
+
+	content = strings.Join(resultLines, "\n")
+
+	return content, nil
+}
+
 // findDefaultSSHKey looks for default SSH keys
 func findDefaultSSHKey() string {
 	homeDir, err := os.UserHomeDir()

src/main_test.go

@@ -5,6 +5,7 @@ package main
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 )
 
@@ -71,15 +72,25 @@ func TestFindDefaultSSHKeyNotFound(t *testing.T) {
 }
 
 func TestLoadConfig(t *testing.T) {
-	// Create a temporary config file
+	// Create a temporary directory and files
 	tempDir := t.TempDir()
-	configPath := filepath.Join(tempDir, "test-config.yaml")
 
-	configContent := `types:
+	// Create device-types.yaml file
-  test-type:
+	deviceTypesPath := filepath.Join(tempDir, "device-types.yaml")
+	deviceTypesContent := `test-type:
   commands:
     - show version
-      - show status
+    - show status`
+
+	err := os.WriteFile(deviceTypesPath, []byte(deviceTypesContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create device-types file: %v", err)
+	}
+
+	// Create main config file with !include
+	configPath := filepath.Join(tempDir, "test-config.yaml")
+	configContent := `types:
+  !include device-types.yaml
 
 devices:
   test-device:
@@ -91,7 +102,7 @@ devices:
       - direct command
 `
 
-	err := os.WriteFile(configPath, []byte(configContent), 0644)
+	err = os.WriteFile(configPath, []byte(configContent), 0644)
 	if err != nil {
 		t.Fatalf("Failed to create test config file: %v", err)
 	}
@@ -285,3 +296,179 @@ func TestRouterBackupCreation(t *testing.T) {
 		})
 	}
 }
+
+// Test !include functionality
+func TestProcessIncludes(t *testing.T) {
+	tempDir := t.TempDir()
+
+	// Create included file
+	includedPath := filepath.Join(tempDir, "included.yaml")
+	includedContent := `test-type:
+  commands:
+    - show version
+    - show status`
+
+	err := os.WriteFile(includedPath, []byte(includedContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create included file: %v", err)
+	}
+
+	// Create main file with !include
+	mainPath := filepath.Join(tempDir, "main.yaml")
+	mainContent := `types:
+  !include included.yaml
+devices:
+  test-device:
+    user: testuser
+    type: test-type`
+
+	err = os.WriteFile(mainPath, []byte(mainContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create main file: %v", err)
+	}
+
+	// Process includes
+	result, err := processIncludes(mainPath)
+	if err != nil {
+		t.Fatalf("Failed to process includes: %v", err)
+	}
+
+	// Check that include was processed
+	if !strings.Contains(result, "show version") {
+		t.Error("Expected included content to be present in result")
+	}
+	if !strings.Contains(result, "show status") {
+		t.Error("Expected included content to be present in result")
+	}
+	if strings.Contains(result, "!include") {
+		t.Error("Expected !include directive to be replaced")
+	}
+}
+
+func TestProcessIncludesWithQuotes(t *testing.T) {
+	tempDir := t.TempDir()
+
+	// Create included file with spaces in name
+	includedPath := filepath.Join(tempDir, "file with spaces.yaml")
+	includedContent := `production-srlinux:
+  commands:
+    - show version`
+
+	err := os.WriteFile(includedPath, []byte(includedContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create included file: %v", err)
+	}
+
+	// Create main file with quoted !include
+	mainPath := filepath.Join(tempDir, "main.yaml")
+	mainContent := `types:
+  !include "file with spaces.yaml"`
+
+	err = os.WriteFile(mainPath, []byte(mainContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create main file: %v", err)
+	}
+
+	// Process includes
+	result, err := processIncludes(mainPath)
+	if err != nil {
+		t.Fatalf("Failed to process includes: %v", err)
+	}
+
+	// Check that include was processed
+	if !strings.Contains(result, "production-srlinux") {
+		t.Error("Expected included content to be present in result")
+	}
+}
+
+func TestProcessIncludesNonexistentFile(t *testing.T) {
+	tempDir := t.TempDir()
+
+	// Create main file with include to nonexistent file
+	mainPath := filepath.Join(tempDir, "main.yaml")
+	mainContent := `types:
+  !include nonexistent.yaml`
+
+	err := os.WriteFile(mainPath, []byte(mainContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create main file: %v", err)
+	}
+
+	// Process includes should fail
+	_, err = processIncludes(mainPath)
+	if err == nil {
+		t.Error("Expected error for nonexistent include file")
+	}
+}
+
+func TestLoadConfigWithIncludes(t *testing.T) {
+	tempDir := t.TempDir()
+
+	// Create device types file
+	typesPath := filepath.Join(tempDir, "types.yaml")
+	typesContent := `srlinux:
+  commands:
+    - show version
+    - show platform linecard
+eos:
+  commands:
+    - show version
+    - show inventory`
+
+	err := os.WriteFile(typesPath, []byte(typesContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create types file: %v", err)
+	}
+
+	// Create main config file with includes
+	mainPath := filepath.Join(tempDir, "config.yaml")
+	mainContent := `types:
+  !include types.yaml
+devices:
+  asw100:
+    user: admin
+    type: srlinux
+  edge-01:
+    user: operator
+    type: eos`
+
+	err = os.WriteFile(mainPath, []byte(mainContent), 0644)
+	if err != nil {
+		t.Fatalf("Failed to create main config file: %v", err)
+	}
+
+	// Load configuration
+	config, err := loadConfig(mainPath)
+	if err != nil {
+		t.Fatalf("Failed to load config with includes: %v", err)
+	}
+
+	// Verify types were loaded correctly
+	if len(config.Types) != 2 {
+		t.Errorf("Expected 2 types, got %d", len(config.Types))
+	}
+
+	srlinuxType, exists := config.Types["srlinux"]
+	if !exists {
+		t.Error("Expected 'srlinux' type to exist")
+	}
+	if len(srlinuxType.Commands) != 2 {
+		t.Errorf("Expected 2 commands for srlinux type, got %d", len(srlinuxType.Commands))
+	}
+
+	// Verify devices were loaded correctly
+	if len(config.Devices) != 2 {
+		t.Errorf("Expected 2 devices, got %d", len(config.Devices))
+	}
+
+	asw100, exists := config.Devices["asw100"]
+	if !exists {
+		t.Error("Expected 'asw100' device to exist")
+	}
+	if asw100.User != "admin" {
+		t.Errorf("Expected user 'admin', got '%s'", asw100.User)
+	}
+	if asw100.Type != "srlinux" {
+		t.Errorf("Expected type 'srlinux', got '%s'", asw100.Type)
+	}
+}