diff --git a/.gitignore b/.gitignore index 3cdee3b..31b4742 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ ipng-router-backup -config.yaml +*.yaml # Debian packaging artifacts debian/.debhelper/ diff --git a/README.md b/README.md index 9eb44c6..d394a38 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ SSH-based network device configuration backup tool with support for multiple dev - **Multi-device backup**: Configure multiple routers in YAML - **Device type templates**: Reusable command sets per device type +- **Configuration includes**: Split large configs with `!include` directives - **Flexible authentication**: SSH agent, key files, or password - **Selective execution**: Backup specific devices with `--host` flags - **Professional CLI**: Standard flags, version info, and help @@ -24,24 +25,30 @@ make build ### Basic Usage -1. **Create configuration file** (`config.yaml`): +1. **Create configuration files**: -```yaml -types: - srlinux: - commands: - - show version - - show platform linecard - - info flat from running + **Main config** (`config.yaml`): + ```yaml + !include device-types.yaml -devices: - asw100: - user: admin - type: srlinux - asw120: - user: admin - type: srlinux -``` + devices: + asw100: + user: admin + type: srlinux + asw120: + user: admin + type: srlinux + ``` + + **Device types** (`device-types.yaml`): + ```yaml + types: + srlinux: + commands: + - show version + - show platform linecard + - info flat from running + ``` 2. **Run backup**: diff --git a/debian/rules b/debian/rules index 667d56a..03cb931 100755 --- a/debian/rules +++ b/debian/rules @@ -12,6 +12,7 @@ override_dh_auto_install: mkdir -p debian/ipng-router-backup/usr/share/man/man1 cp ipng-router-backup debian/ipng-router-backup/usr/bin/ cp docs/config.yaml.example debian/ipng-router-backup/etc/ipng-router-backup/config.yaml.example + cp docs/device-types.yaml debian/ipng-router-backup/etc/ipng-router-backup/device-types.yaml cp docs/router_backup.1 debian/ipng-router-backup/usr/share/man/man1/ipng-router-backup.1 gzip debian/ipng-router-backup/usr/share/man/man1/ipng-router-backup.1 diff --git a/docs/DETAILS.md b/docs/DETAILS.md index c3f8e39..6b02855 100644 --- a/docs/DETAILS.md +++ b/docs/DETAILS.md @@ -8,6 +8,7 @@ IPng Networks Router Backup is a SSH-based network device configuration backup t - **Multi-device support**: Backup multiple routers in a single run - **Device type templates**: Define command sets per device type +- **Configuration includes**: Split large configurations with `!include` directives - **Flexible authentication**: SSH agent, key files, or password authentication - **Selective execution**: Target specific devices with `--host` flags - **Automatic file organization**: Output files named by hostname @@ -16,10 +17,35 @@ IPng Networks Router Backup is a SSH-based network device configuration backup t ## Configuration File Format -The tool uses a YAML configuration file with two main sections: `types` and `devices`. +The tool uses a YAML configuration file with two main sections: `types` and `devices`. The configuration supports `!include` directives for organizing large configurations across multiple files. ### Complete Example +**Main configuration** (`config.yaml`): +```yaml +!include device-types.yaml + +devices: + asw100: + user: admin + type: srlinux + + asw120: + user: netops + type: srlinux + + core-01: + user: admin + type: eos + + edge-router: + user: operator + commands: + - show version + - show ip route summary +``` + +**Device types file** (`device-types.yaml`): ```yaml types: srlinux: @@ -29,39 +55,20 @@ types: - show platform fan-tray - show platform power-supply - info flat from running - + eos: commands: - show version - show inventory - show env power - show running-config - + centec: commands: - show version | exc uptime - show boot images - show transceiver - show running-config - -devices: - asw100: - user: admin - type: srlinux - - asw120: - user: netops - type: srlinux - - core-01: - user: admin - type: eos - - edge-router: - user: operator - commands: - - show version - - show ip route summary ``` ### Configuration Fields @@ -89,6 +96,61 @@ devices: - Type references must exist in the `types` section - Commands can be specified either via type reference or directly per device +### Include Directive Support + +The configuration supports `!include` directives for splitting large configurations into multiple files: + +```yaml +# Main config.yaml +!include device-types.yaml + +devices: + production-device: + user: admin + type: srlinux +``` + +**Include Features:** +- **One level deep**: Included files cannot contain their own `!include` directives +- **Relative paths**: Paths are relative to the including file's directory +- **Absolute paths**: Fully qualified paths are supported +- **Quoted paths**: Use quotes for paths containing spaces: `!include "file with spaces.yaml"` +- **Proper indentation**: Included content maintains correct YAML indentation + +**Example file structure:** +``` +/etc/ipng-router-backup/ +├── config.yaml # Main configuration with !include +├── device-types.yaml # Device type definitions +└── devices/ + ├── production.yaml # Production device definitions + └── lab.yaml # Lab device definitions +``` + +**Usage patterns:** + +1. **Include device types at top level:** + ```yaml + !include device-types.yaml + + devices: + # device definitions here + ``` + +2. **Include under specific sections:** + ```yaml + types: + !include types/network-devices.yaml + + devices: + !include devices/production.yaml + ``` + +3. **Include files with spaces:** + ```yaml + !include "device types/lab environment.yaml" + ``` + ## Command Line Flags ### Required Flags @@ -109,7 +171,7 @@ devices: ```bash # Basic usage - all devices -ipng-router-backup --config /etc/network-backup/config.yaml +ipng-router-backup --config /etc/ipng-router-backup/config.yaml # Custom output directory ipng-router-backup --config config.yaml --output-dir /backup/network @@ -163,7 +225,7 @@ ipng-router-backup --config config.yaml --key-file ~/.ssh/network_key # Tool automatically checks these default locations: # ~/.ssh/id_rsa -# ~/.ssh/id_ed25519 +# ~/.ssh/id_ed25519 # ~/.ssh/id_ecdsa ``` @@ -255,7 +317,7 @@ BACKUP_DIR="/backup/network/$(date +%Y%m%d)" mkdir -p "$BACKUP_DIR" ipng-router-backup \ - --config /etc/network-backup/config.yaml \ + --config /etc/ipng-router-backup/config.yaml \ --output-dir "$BACKUP_DIR" # Kill SSH agent @@ -299,6 +361,55 @@ ipng-router-backup \ ## Advanced Usage +### Configuration Organization with Includes + +For large deployments, organize configurations using `!include` directives: + +**Environment-based structure:** +```bash +network-backup/ +├── config.yaml # Main config +├── types/ +│ ├── device-types.yaml # All device types +│ └── vendor-specific.yaml # Vendor-specific commands +├── environments/ +│ ├── production.yaml # Production devices +│ ├── staging.yaml # Staging devices +│ └── lab.yaml # Lab devices +└── sites/ + ├── datacenter-east.yaml # East datacenter devices + └── datacenter-west.yaml # West datacenter devices +``` + +**Main configuration** (`config.yaml`): +```yaml +!include types/device-types.yaml + +devices: + # Production environment + !include environments/production.yaml + + # Lab environment + !include environments/lab.yaml +``` + +**Production devices** (`environments/production.yaml`): +```yaml +# Production SR Linux switches +prod-asw100: + user: netops + type: srlinux + +prod-asw120: + user: netops + type: srlinux + +# Production EOS devices +prod-core-01: + user: netops + type: eos +``` + ### Integration with Git ```bash @@ -324,9 +435,9 @@ types: production-srlinux: commands: - show version - - show system information + - show system information - info flat from running - + lab-srlinux: commands: - show version @@ -336,7 +447,7 @@ devices: prod-asw100: user: readonly type: production-srlinux - + lab-asw100: user: admin type: lab-srlinux diff --git a/docs/config.yaml.example b/docs/config.yaml.example index ca44d05..cf949cf 100644 --- a/docs/config.yaml.example +++ b/docs/config.yaml.example @@ -5,64 +5,29 @@ # Copy this file to a location of your choice and modify for your environment. # # Usage: ipng-router-backup --config /path/to/your/config.yaml +# +# YAML !include Support: +# You can split large configurations into multiple files using !include directives. +# Examples: +# !include device-types.yaml +# !include devices/production.yaml +# !include "devices/lab environment.yaml" # Use quotes for paths with spaces -# Device Types Section -# Define reusable command sets for different types of network equipment -types: - # Nokia SR Linux devices - srlinux: - commands: - - show version # System version and build info - - show platform linecard # Line card information - - show platform fan-tray # Fan status and health - - show platform power-supply # Power supply status - - info flat from running # Full running configuration +# Include device types from separate file +!include device-types.yaml - # Arista EOS devices - eos: - commands: - - show version # System version information - - show inventory # Hardware inventory - - show env power # Power supply status - - show running-config # Complete running configuration - - # Centec switches - centec: - commands: - - show version | exc uptime # Version info without uptime line - - show boot images # Boot image information - - show transceiver # SFP/transceiver status - - show running-config # Running configuration - - # Cisco IOS/IOS-XE devices - cisco-ios: - commands: - - show version # IOS version and hardware info - - show inventory # Hardware inventory details - - show running-config # Complete configuration - - show ip interface brief # Interface IP summary - - show cdp neighbors # CDP neighbor information - - # Juniper devices - junos: - commands: - - show version # Software and hardware version - - show chassis hardware # Chassis hardware details - - show configuration | display set # Configuration in set format - - show interfaces terse # Interface status summary - -# Devices Section +# Devices Section # Define individual network devices to backup devices: # Core switches (SR Linux) asw100: user: admin # SSH username type: srlinux # Reference to type above - + asw120: user: netops # Different user per device if needed type: srlinux - + asw121: user: admin type: srlinux @@ -71,16 +36,16 @@ devices: csw150: user: admin type: centec - + csw151: - user: admin + user: admin type: centec # Edge routers (Arista EOS) edge-01: user: automation type: eos - + edge-02: user: automation type: eos @@ -100,7 +65,7 @@ devices: type: cisco-ios # Configuration Tips: -# +# # 1. Authentication Priority (automatic): # - SSH Agent (if SSH_AUTH_SOCK environment variable is set) # - SSH Key file (--key-file flag or default locations) @@ -109,10 +74,10 @@ devices: # 2. Running the backup: # # Backup all devices # ipng-router-backup --config /etc/ipng-router-backup/config.yaml -# -# # Backup specific devices only +# +# # Backup specific devices only # ipng-router-backup --config config.yaml --host asw100 --host edge-01 -# +# # # Custom output directory # ipng-router-backup --config config.yaml --output-dir /backup/$(date +%Y%m%d) # @@ -130,4 +95,4 @@ devices: # 5. Error handling: # - If a device is unreachable, the tool continues with other devices # - Check tool output for connection or authentication failures -# - Use --host flag to test individual devices +# - Use --host flag to test individual devices \ No newline at end of file diff --git a/src/main.go b/src/main.go index e20254c..585346a 100644 --- a/src/main.go +++ b/src/main.go @@ -9,6 +9,8 @@ import ( "net" "os" "path/filepath" + "regexp" + "strings" "time" "github.com/spf13/cobra" @@ -180,15 +182,15 @@ func (rb *RouterBackup) Disconnect() { } } -// loadConfig loads the YAML configuration file +// loadConfig loads the YAML configuration file with !include support func loadConfig(configPath string) (*Config, error) { - data, err := ioutil.ReadFile(configPath) + processedYAML, err := processIncludes(configPath) if err != nil { - return nil, fmt.Errorf("failed to read config file %s: %v", configPath, err) + return nil, fmt.Errorf("failed to process includes: %v", err) } var config Config - err = yaml.Unmarshal(data, &config) + err = yaml.Unmarshal([]byte(processedYAML), &config) if err != nil { return nil, fmt.Errorf("failed to parse YAML: %v", err) } @@ -196,6 +198,75 @@ func loadConfig(configPath string) (*Config, error) { return &config, nil } +// processIncludes processes YAML files with !include directives (one level deep) +func processIncludes(filePath string) (string, error) { + // Read the file + data, err := ioutil.ReadFile(filePath) + if err != nil { + return "", fmt.Errorf("failed to read file %s: %v", filePath, err) + } + + content := string(data) + + // Process !include directives + // Match patterns like: !include path/to/file.yaml (excluding commented lines) + includeRegex := regexp.MustCompile(`(?m)^(\s*)!include\s+(.+)$`) + + baseDir := filepath.Dir(filePath) + + // Process includes line by line to avoid conflicts + lines := strings.Split(content, "\n") + var resultLines []string + + for _, line := range lines { + // Check if this line matches our include pattern + if match := includeRegex.FindStringSubmatch(line); match != nil { + leadingWhitespace := match[1] + includePath := strings.TrimSpace(match[2]) + + // Skip commented lines + if strings.Contains(strings.TrimSpace(line), "#") && strings.Index(strings.TrimSpace(line), "#") < strings.Index(strings.TrimSpace(line), "!include") { + resultLines = append(resultLines, line) + continue + } + + // Remove quotes if present + includePath = strings.Trim(includePath, "\"'") + + // Make path relative to current config file + if !filepath.IsAbs(includePath) { + includePath = filepath.Join(baseDir, includePath) + } + + // Read the included file + includedData, err := ioutil.ReadFile(includePath) + if err != nil { + return "", fmt.Errorf("failed to read include file %s: %v", includePath, err) + } + + // Use the captured leading whitespace as indentation prefix + indentPrefix := leadingWhitespace + + // Indent each line of included content to match the !include line's indentation + includedLines := strings.Split(string(includedData), "\n") + for _, includeLine := range includedLines { + if strings.TrimSpace(includeLine) == "" { + resultLines = append(resultLines, "") + } else { + resultLines = append(resultLines, indentPrefix+includeLine) + } + } + } else { + // Regular line, just copy it + resultLines = append(resultLines, line) + } + } + + content = strings.Join(resultLines, "\n") + + return content, nil +} + // findDefaultSSHKey looks for default SSH keys func findDefaultSSHKey() string { homeDir, err := os.UserHomeDir() diff --git a/src/main_test.go b/src/main_test.go index b97a128..dc15707 100644 --- a/src/main_test.go +++ b/src/main_test.go @@ -5,6 +5,7 @@ package main import ( "os" "path/filepath" + "strings" "testing" ) @@ -71,15 +72,25 @@ func TestFindDefaultSSHKeyNotFound(t *testing.T) { } func TestLoadConfig(t *testing.T) { - // Create a temporary config file + // Create a temporary directory and files tempDir := t.TempDir() - configPath := filepath.Join(tempDir, "test-config.yaml") + // Create device-types.yaml file + deviceTypesPath := filepath.Join(tempDir, "device-types.yaml") + deviceTypesContent := `test-type: + commands: + - show version + - show status` + + err := os.WriteFile(deviceTypesPath, []byte(deviceTypesContent), 0644) + if err != nil { + t.Fatalf("Failed to create device-types file: %v", err) + } + + // Create main config file with !include + configPath := filepath.Join(tempDir, "test-config.yaml") configContent := `types: - test-type: - commands: - - show version - - show status + !include device-types.yaml devices: test-device: @@ -91,7 +102,7 @@ devices: - direct command ` - err := os.WriteFile(configPath, []byte(configContent), 0644) + err = os.WriteFile(configPath, []byte(configContent), 0644) if err != nil { t.Fatalf("Failed to create test config file: %v", err) } @@ -285,3 +296,179 @@ func TestRouterBackupCreation(t *testing.T) { }) } } + +// Test !include functionality +func TestProcessIncludes(t *testing.T) { + tempDir := t.TempDir() + + // Create included file + includedPath := filepath.Join(tempDir, "included.yaml") + includedContent := `test-type: + commands: + - show version + - show status` + + err := os.WriteFile(includedPath, []byte(includedContent), 0644) + if err != nil { + t.Fatalf("Failed to create included file: %v", err) + } + + // Create main file with !include + mainPath := filepath.Join(tempDir, "main.yaml") + mainContent := `types: + !include included.yaml +devices: + test-device: + user: testuser + type: test-type` + + err = os.WriteFile(mainPath, []byte(mainContent), 0644) + if err != nil { + t.Fatalf("Failed to create main file: %v", err) + } + + // Process includes + result, err := processIncludes(mainPath) + if err != nil { + t.Fatalf("Failed to process includes: %v", err) + } + + // Check that include was processed + if !strings.Contains(result, "show version") { + t.Error("Expected included content to be present in result") + } + if !strings.Contains(result, "show status") { + t.Error("Expected included content to be present in result") + } + if strings.Contains(result, "!include") { + t.Error("Expected !include directive to be replaced") + } +} + +func TestProcessIncludesWithQuotes(t *testing.T) { + tempDir := t.TempDir() + + // Create included file with spaces in name + includedPath := filepath.Join(tempDir, "file with spaces.yaml") + includedContent := `production-srlinux: + commands: + - show version` + + err := os.WriteFile(includedPath, []byte(includedContent), 0644) + if err != nil { + t.Fatalf("Failed to create included file: %v", err) + } + + // Create main file with quoted !include + mainPath := filepath.Join(tempDir, "main.yaml") + mainContent := `types: + !include "file with spaces.yaml"` + + err = os.WriteFile(mainPath, []byte(mainContent), 0644) + if err != nil { + t.Fatalf("Failed to create main file: %v", err) + } + + // Process includes + result, err := processIncludes(mainPath) + if err != nil { + t.Fatalf("Failed to process includes: %v", err) + } + + // Check that include was processed + if !strings.Contains(result, "production-srlinux") { + t.Error("Expected included content to be present in result") + } +} + +func TestProcessIncludesNonexistentFile(t *testing.T) { + tempDir := t.TempDir() + + // Create main file with include to nonexistent file + mainPath := filepath.Join(tempDir, "main.yaml") + mainContent := `types: + !include nonexistent.yaml` + + err := os.WriteFile(mainPath, []byte(mainContent), 0644) + if err != nil { + t.Fatalf("Failed to create main file: %v", err) + } + + // Process includes should fail + _, err = processIncludes(mainPath) + if err == nil { + t.Error("Expected error for nonexistent include file") + } +} + +func TestLoadConfigWithIncludes(t *testing.T) { + tempDir := t.TempDir() + + // Create device types file + typesPath := filepath.Join(tempDir, "types.yaml") + typesContent := `srlinux: + commands: + - show version + - show platform linecard +eos: + commands: + - show version + - show inventory` + + err := os.WriteFile(typesPath, []byte(typesContent), 0644) + if err != nil { + t.Fatalf("Failed to create types file: %v", err) + } + + // Create main config file with includes + mainPath := filepath.Join(tempDir, "config.yaml") + mainContent := `types: + !include types.yaml +devices: + asw100: + user: admin + type: srlinux + edge-01: + user: operator + type: eos` + + err = os.WriteFile(mainPath, []byte(mainContent), 0644) + if err != nil { + t.Fatalf("Failed to create main config file: %v", err) + } + + // Load configuration + config, err := loadConfig(mainPath) + if err != nil { + t.Fatalf("Failed to load config with includes: %v", err) + } + + // Verify types were loaded correctly + if len(config.Types) != 2 { + t.Errorf("Expected 2 types, got %d", len(config.Types)) + } + + srlinuxType, exists := config.Types["srlinux"] + if !exists { + t.Error("Expected 'srlinux' type to exist") + } + if len(srlinuxType.Commands) != 2 { + t.Errorf("Expected 2 commands for srlinux type, got %d", len(srlinuxType.Commands)) + } + + // Verify devices were loaded correctly + if len(config.Devices) != 2 { + t.Errorf("Expected 2 devices, got %d", len(config.Devices)) + } + + asw100, exists := config.Devices["asw100"] + if !exists { + t.Error("Expected 'asw100' device to exist") + } + if asw100.User != "admin" { + t.Errorf("Expected user 'admin', got '%s'", asw100.User) + } + if asw100.Type != "srlinux" { + t.Errorf("Expected type 'srlinux', got '%s'", asw100.Type) + } +}