diff --git a/cmd/standalone.go b/cmd/standalone.go index 0d8ad9c..5b7cffa 100644 --- a/cmd/standalone.go +++ b/cmd/standalone.go @@ -64,7 +64,8 @@ Run as an agent (default): nudgebee-forager --config /etc/nudgebee/forager.yaml Run discovery directly, without a relay or an account: - nudgebee-forager sweep --cidr 10.0.1.0/24 [--ports 22] [--rate-pps 100] + nudgebee-forager sweep --cidr 10.0.1.0/24 [--ports 22] [--rate-pps 100] \ + [--user nudgebee-ro --key ~/.ssh/id_ed25519] nudgebee-forager inventory --cidr 10.0.1.0/24 --targets 10.0.1.5,10.0.1.6 \ --user nudgebee-ro --key ~/.ssh/id_ed25519 \ --pack ./linux-inventory-v2.yaml --pack-key @@ -129,6 +130,13 @@ func cmdSweep(args []string) error { ratePPS := fs.Int("rate-pps", 100, "probes per second") timeoutMs := fs.Int("timeout-ms", 1000, "per-probe timeout") exclude := fs.String("exclude", "", "comma-separated addresses or CIDRs to skip") + // SSH credentials are optional here, unlike inventory: a sweep with none + // still reports presence/MAC/RDNS, just without the cloud-identity probe + // (see pkg/proxy/discovery/cloud_identity.go) that needs to log in. + user := fs.String("user", "nudgebee-ro", "SSH username, enables the cloud-identity probe when set alongside --key/--password-env") + keyFile := fs.String("key", "", "path to SSH private key, enables the cloud-identity probe") + passwordEnv := fs.String("password-env", "", "environment variable holding the SSH password, enables the cloud-identity probe") + sshPort := fs.Int("ssh-port", 22, "SSH port the cloud-identity probe connects on") verbose := fs.Bool("v", false, "log progress to stderr") if err := fs.Parse(args); err != nil { return err @@ -136,6 +144,9 @@ func cmdSweep(args []string) error { if *cidr == "" { return fmt.Errorf("--cidr is required") } + if *keyFile != "" && *passwordEnv != "" { + return fmt.Errorf("--key and --password-env are mutually exclusive") + } portList, err := parsePorts(*ports) if err != nil { @@ -152,12 +163,32 @@ func cmdSweep(args []string) error { params["exclusions"] = toAnySlice(splitList(*exclude)) } + creds := map[string]string{} + if *keyFile != "" || *passwordEnv != "" { + creds["username"] = *user + if *keyFile != "" { + key, err := os.ReadFile(*keyFile) + if err != nil { + return fmt.Errorf("reading --key: %w", err) + } + creds["private_key"] = string(key) + } + if *passwordEnv != "" { + pw := os.Getenv(*passwordEnv) + if pw == "" { + return fmt.Errorf("environment variable %s is empty", *passwordEnv) + } + creds["password"] = pw + } + } + // The swept CIDR is also the scope ceiling, so a standalone run cannot // reach further than what was asked for. return runAction(map[string]any{ "allowed_cidrs": []any{*cidr}, "max_rate_pps": float64(*ratePPS), - }, map[string]string{}, "discovery_sweep", params, *verbose) + "port": float64(*sshPort), + }, creds, "discovery_sweep", params, *verbose) } func cmdInventory(args []string) error { diff --git a/cmd/standalone_test.go b/cmd/standalone_test.go index 4460110..d8b71ef 100644 --- a/cmd/standalone_test.go +++ b/cmd/standalone_test.go @@ -87,6 +87,15 @@ func TestCmdSweep_RequiresCIDR(t *testing.T) { } } +// --key and --password-env both name an auth method; accepting both silently +// picking one would surprise whichever one lost. +func TestCmdSweep_RejectsBothKeyAndPasswordEnv(t *testing.T) { + err := cmdSweep([]string{"--cidr", "10.0.0.0/24", "--key", "/dev/null", "--password-env", "SSH_PW"}) + if err == nil { + t.Fatal("accepted both --key and --password-env") + } +} + // The pack version comes from the file so the caller does not have to repeat // it, and a mismatch would make the proxy refuse the pack. func TestStagePack(t *testing.T) { diff --git a/docs/cli.md b/docs/cli.md index ea29e0d..fd24219 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -32,10 +32,14 @@ Results print to stdout as JSON, logs to stderr, so output pipes into `jq`. ## sweep — what is on this network Probes every address in a range and reports what answered. No credentials -needed, and nothing is read from the machines themselves. +needed, and nothing is read from the machines themselves — unless `--user` +is paired with `--key`/`--password-env`, in which case each host that +answers on the SSH port also gets a cloud-instance-identity probe (AWS/GCP/ +Azure instance metadata, read over SSH — see `pkg/proxy/discovery/cloud_identity.go`). ```bash ./forager sweep --cidr 192.168.1.0/24 --ports 22 +./forager sweep --cidr 192.168.1.0/24 --user nudgebee-ro --key ~/.ssh/id_ed25519 ``` | Flag | Default | | @@ -45,6 +49,10 @@ needed, and nothing is read from the machines themselves. | `--rate-pps` | `100` | Probes per second | | `--timeout-ms` | `1000` | Per-probe timeout | | `--exclude` | | Addresses or CIDRs to skip entirely | +| `--user` | `nudgebee-ro` | SSH username for the cloud-identity probe | +| `--key` | | Path to SSH private key — enables the cloud-identity probe | +| `--password-env` | | Env var holding the SSH password, instead of `--key` — enables the cloud-identity probe | +| `--ssh-port` | `22` | Port the cloud-identity probe connects on | | `-v` | | Log progress to stderr | ```json @@ -56,7 +64,8 @@ needed, and nothing is read from the machines themselves. "duration_seconds": 3.2, "hosts": [ {"ip": "192.168.1.50", "open_ports": [22], "mac": "aa:bb:cc:dd:ee:ff", - "rdns": "web-01.lan", "sources": ["tcp", "arp"]} + "rdns": "web-01.lan", "sources": ["tcp", "arp"], + "cloud_identity": "provider=aws\ninstance_id=i-0aab26d051729d673\nregion=us-east-1\npublic_ip=54.1.2.3\n"} ] } ``` @@ -64,7 +73,11 @@ needed, and nothing is read from the machines themselves. `mac` appears only for hosts on the same network segment — anything reached through a router will not have one, and neither will the machine you are running from, since a host does not ARP for its own address. `rdns` appears -only when reverse DNS resolves. Neither absence is an error. +only when reverse DNS resolves. `cloud_identity` appears only when SSH +credentials were supplied, the host answered on the SSH port, and it's +actually running on a cloud whose metadata service answered — a bare-metal +host, a non-SSH host, or a run with no `--user`/`--key` all leave it absent. +None of these are errors. `addresses_scanned` excludes the network and broadcast addresses, so a `/24` scans 254 rather than 256. diff --git a/pkg/proxy/discovery/cloud_identity.go b/pkg/proxy/discovery/cloud_identity.go new file mode 100644 index 0000000..2939ba3 --- /dev/null +++ b/pkg/proxy/discovery/cloud_identity.go @@ -0,0 +1,120 @@ +package discovery + +import ( + "context" + "slices" + "strings" + "sync" + "time" +) + +// Deliberately short relative to inventory's timeouts: this runs against +// every host a sweep finds (a much larger set than a filtered inventory +// batch typically is), and a slow/unreachable host must not meaningfully +// delay the sweep response. +const ( + cloudIdentityDialTimeout = 5 * time.Second + cloudIdentityHostTimeout = 10 * time.Second + cloudIdentityCommandTimeout = 8 * time.Second + cloudIdentityMaxOutputBytes = 4096 // a handful of key=value lines +) + +// cloudIdentityProbeCmd detects which cloud (if any) a host runs on by +// querying each provider's instance metadata service in turn, and prints the +// bare facts needed to identify the instance — never a full document. IMDS +// endpoints are link-local and only answer requests originating from the +// instance itself, so this has to run over SSH on the host, the same as +// every other command this package issues; there is no way to query it +// remotely from the forager's own process. +// +// Every field is fetched as plain text (Azure via format=text, including on +// indexed array leaves) so no JSON parser is required on the target host. +// Each attempt is capped at 1s and short-circuits on the first cloud that +// answers, so a non-cloud host pays at most ~3s total before this prints +// nothing. Output is unparsed key=value lines, one block for whichever +// provider matched; parsing lives server-side (see resolveOneTarget in +// nudgebee-enterprise's vmpackage/resource_match.go) so a parser fix never +// requires an agent release. +const cloudIdentityProbeCmd = ` +TOKEN=$(curl -s -m 1 -X PUT http://169.254.169.254/latest/api/token -H "X-aws-ec2-metadata-token-ttl-seconds: 60" 2>/dev/null) +AWS_ID=$(curl -s -m 1 -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-id 2>/dev/null) +if [ -n "$AWS_ID" ]; then + echo "provider=aws" + echo "instance_id=$AWS_ID" + echo "region=$(curl -s -m 1 -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/placement/region 2>/dev/null)" + echo "public_ip=$(curl -s -m 1 -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/public-ipv4 2>/dev/null)" + exit 0 +fi +GCP_ID=$(curl -s -m 1 -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/id 2>/dev/null) +if [ -n "$GCP_ID" ]; then + ZONE_PATH=$(curl -s -m 1 -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/zone 2>/dev/null) + echo "provider=gcp" + echo "instance_id=$GCP_ID" + echo "zone=${ZONE_PATH##*/}" + echo "public_ip=$(curl -s -m 1 -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip 2>/dev/null)" + exit 0 +fi +AZ_NAME=$(curl -s -m 1 -H "Metadata: true" "http://169.254.169.254/metadata/instance/compute/name?api-version=2021-02-01&format=text" 2>/dev/null) +if [ -n "$AZ_NAME" ]; then + echo "provider=azure" + echo "subscription_id=$(curl -s -m 1 -H "Metadata: true" "http://169.254.169.254/metadata/instance/compute/subscriptionId?api-version=2021-02-01&format=text" 2>/dev/null)" + echo "resource_group=$(curl -s -m 1 -H "Metadata: true" "http://169.254.169.254/metadata/instance/compute/resourceGroupName?api-version=2021-02-01&format=text" 2>/dev/null)" + echo "name=$AZ_NAME" + echo "location=$(curl -s -m 1 -H "Metadata: true" "http://169.254.169.254/metadata/instance/compute/location?api-version=2021-02-01&format=text" 2>/dev/null)" + echo "public_ip=$(curl -s -m 1 -H "Metadata: true" "http://169.254.169.254/metadata/instance/network/interface/0/ipv4/ipAddress/0/publicIpAddress?api-version=2021-02-01&format=text" 2>/dev/null)" +fi +` + +// enrichCloudIdentity fills in CloudIdentity on every host in hosts that has +// the SSH port open, using cfg.sshClientConfig. A no-op when no SSH +// credentials are configured on this datasource (cfg.sshClientConfig == nil) +// — sweep-only datasources keep behaving exactly as before. One host's +// dial/auth/command failure never affects another's, and never fails the +// sweep as a whole: an unreachable or non-SSH host simply keeps an empty +// CloudIdentity, same tolerance enrichFromARP/enrichRDNS already have for +// hosts they cannot enrich. +func enrichCloudIdentity(ctx context.Context, hosts []SweepHost, cfg execConfig) { + if cfg.sshClientConfig == nil { + return + } + + concurrency := cfg.concurrency + if concurrency <= 0 { + concurrency = defaultConcurrency + } + + var wg sync.WaitGroup + sem := make(chan struct{}, concurrency) + + for i := range hosts { + if !slices.Contains(hosts[i].OpenPorts, cfg.port) { + continue + } + wg.Add(1) + go func(idx int) { + defer wg.Done() + select { + case sem <- struct{}{}: + defer func() { <-sem }() + case <-ctx.Done(): + return + } + + hostCtx, cancel := context.WithTimeout(ctx, cfg.hostTimeout) + defer cancel() + + client, err := dial(hostCtx, hosts[idx].IP, cfg) + if err != nil { + return + } + defer func() { _ = client.Close() }() + + out, _, err := runCommand(hostCtx, client, cloudIdentityProbeCmd, cfg) + if err != nil || strings.TrimSpace(out) == "" { + return + } + hosts[idx].CloudIdentity = out + }(i) + } + wg.Wait() +} diff --git a/pkg/proxy/discovery/cloud_identity_test.go b/pkg/proxy/discovery/cloud_identity_test.go new file mode 100644 index 0000000..0d9028d --- /dev/null +++ b/pkg/proxy/discovery/cloud_identity_test.go @@ -0,0 +1,87 @@ +package discovery + +import ( + "context" + "strings" + "testing" +) + +func TestEnrichCloudIdentity_PopulatesFromProbe(t *testing.T) { + srv := newFakeSSHServer(t, map[string]string{ + cloudIdentityProbeCmd: "provider=aws\ninstance_id=i-0aab26d051729d673\nregion=us-east-1\npublic_ip=\n", + }) + cfg := testExecConfig(srv.port(), 5) + hosts := []SweepHost{{IP: "127.0.0.1", OpenPorts: []int{cfg.port}}} + + enrichCloudIdentity(context.Background(), hosts, cfg) + + if !strings.Contains(hosts[0].CloudIdentity, "provider=aws") { + t.Errorf("expected CloudIdentity to contain the probe output, got %q", hosts[0].CloudIdentity) + } +} + +func TestEnrichCloudIdentity_NoSSHCredsIsNoop(t *testing.T) { + // sshClientConfig is nil: mirrors a sweep-only datasource with no SSH + // credentials configured at all — must not attempt anything. + cfg := execConfig{port: 22} + hosts := []SweepHost{{IP: "127.0.0.1", OpenPorts: []int{22}}} + + enrichCloudIdentity(context.Background(), hosts, cfg) + + if hosts[0].CloudIdentity != "" { + t.Error("expected no-op when no SSH credentials are configured") + } +} + +func TestEnrichCloudIdentity_SkipsHostsWithoutSSHPortOpen(t *testing.T) { + // Registers a truthy response so that if the port filter were broken and + // this dialed anyway, the test would catch it via a populated CloudIdentity + // rather than passing vacuously. + srv := newFakeSSHServer(t, map[string]string{ + cloudIdentityProbeCmd: "provider=aws\ninstance_id=i-abc\n", + }) + cfg := testExecConfig(srv.port(), 5) + hosts := []SweepHost{{IP: "127.0.0.1", OpenPorts: []int{3389}}} // RDP only, no SSH port + + enrichCloudIdentity(context.Background(), hosts, cfg) + + if hosts[0].CloudIdentity != "" { + t.Error("must not probe a host that has no SSH port open") + } +} + +func TestEnrichCloudIdentity_NonCloudHostStaysEmpty(t *testing.T) { + // Every provider's metadata service timed out on a bare-metal/non-cloud + // host — the real command prints nothing in that case. + srv := newFakeSSHServer(t, map[string]string{ + cloudIdentityProbeCmd: "", + }) + cfg := testExecConfig(srv.port(), 5) + hosts := []SweepHost{{IP: "127.0.0.1", OpenPorts: []int{cfg.port}}} + + enrichCloudIdentity(context.Background(), hosts, cfg) + + if hosts[0].CloudIdentity != "" { + t.Error("empty probe output must leave CloudIdentity empty") + } +} + +func TestEnrichCloudIdentity_OneHostFailureDoesNotAffectAnother(t *testing.T) { + srv := newFakeSSHServer(t, map[string]string{ + cloudIdentityProbeCmd: "provider=gcp\ninstance_id=123\n", + }) + cfg := testExecConfig(srv.port(), 5) + hosts := []SweepHost{ + {IP: "127.0.0.1", OpenPorts: []int{cfg.port}}, + {IP: "203.0.113.254", OpenPorts: []int{cfg.port}}, // unreachable (TEST-NET-3) + } + + enrichCloudIdentity(context.Background(), hosts, cfg) + + if !strings.Contains(hosts[0].CloudIdentity, "provider=gcp") { + t.Errorf("reachable host must still be enriched, got %q", hosts[0].CloudIdentity) + } + if hosts[1].CloudIdentity != "" { + t.Error("unreachable host must stay empty, not error") + } +} diff --git a/pkg/proxy/discovery/proxy.go b/pkg/proxy/discovery/proxy.go index 107478b..2dbca48 100644 --- a/pkg/proxy/discovery/proxy.go +++ b/pkg/proxy/discovery/proxy.go @@ -333,6 +333,9 @@ func (p *Proxy) handleSweep(ctx context.Context, req *proxy.ActionRequest) (*pro p.mu.RLock() maxRate := p.cfg.MaxRatePPS configuredCIDRs := p.allowedNets + sshCfg := p.sshConfig + sshPort := p.cfg.Port + concurrency := p.cfg.Concurrency p.mu.RUnlock() cfg, err := parseSweepParams(req.Params, maxRate) @@ -357,6 +360,26 @@ func (p *Proxy) handleSweep(ctx context.Context, req *proxy.ActionRequest) (*pro return nil, fmt.Errorf("discovery_sweep: %w", err) } + // Best-effort cloud-instance-identity probe for whatever this sweep just + // found — a no-op when this datasource has no SSH credentials configured + // (sweep-only datasources keep today's behavior exactly). This is the + // strongest signal the server has for matching a swept host against an + // existing cloud-collector-synced asset, so it belongs on the sweep + // response itself rather than waiting on a separate discovery_inventory + // round that may never run. + if sshCfg != nil { + identityCfg := execConfig{ + port: sshPort, + concurrency: concurrency, + hostTimeout: cloudIdentityHostTimeout, + commandTimeout: cloudIdentityCommandTimeout, + maxOutputBytes: cloudIdentityMaxOutputBytes, + dialTimeout: cloudIdentityDialTimeout, + sshClientConfig: sshCfg, + } + enrichCloudIdentity(ctx, result.Hosts, identityCfg) + } + p.logger.Info("discovery sweep complete", "cidrs", result.CIDRs, "scanned", result.Scanned, diff --git a/pkg/proxy/discovery/sweep.go b/pkg/proxy/discovery/sweep.go index 752ac75..f875d31 100644 --- a/pkg/proxy/discovery/sweep.go +++ b/pkg/proxy/discovery/sweep.go @@ -33,6 +33,13 @@ type SweepHost struct { RDNS string `json:"rdns,omitempty"` OpenPorts []int `json:"open_ports,omitempty"` Sources []string `json:"sources"` // tcp, arp + + // CloudIdentity is cloudIdentityProbeCmd's raw, unparsed output (see + // cloud_identity.go) when this datasource has SSH credentials configured + // and the host answered on the SSH port — empty otherwise (no + // credentials, host unreachable/not SSH, or a non-cloud host that timed + // out on every provider's metadata service). + CloudIdentity string `json:"cloud_identity,omitempty"` } // SweepResult is the response to a discovery_sweep action.