snowpick/tableapi.go

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"sync"
)

// TableAPIResult holds the result of probing a single table via the REST API.
type TableAPIResult struct {
	Table        string        `json:"table"`
	Status       WidgetStatus  `json:"status"`
	RecordCount  int           `json:"record_count"`
	SampleCount  int           `json:"sample_count,omitempty"`
	Fields       []string      `json:"fields,omitempty"`
	Samples      []TableSample `json:"samples,omitempty"`
	Hints        []Hint        `json:"hints,omitempty"`
	Reproduction *Reproduction `json:"reproduction,omitempty"`
	Error        string        `json:"error,omitempty"`
}

type tableAPIResponse struct {
	Result []map[string]any `json:"result"`
}

// DefaultTableAPITables are high-value tables to check via the REST API.
var DefaultTableAPITables = []string{
	"sys_user",
	"incident",
	"kb_knowledge",
	"sys_attachment",
	"oauth_entity",
	"sc_cat_item",
	"cmdb_ci",
	"alm_asset",
	"task",
	"customer_contact",
	"cmn_department",
	"sys_properties",
}

// ProbeTableAPI checks whether a table is accessible via /api/now/table/{table}.
func ProbeTableAPI(ctx context.Context, session *Session, table string, limit int, client *http.Client) *TableAPIResult {
	result := &TableAPIResult{Table: table}

	u := url.URL{
		Scheme: "https",
		Host:   session.Target,
		Path:   "/api/now/table/" + table,
	}
	q := u.Query()
	q.Set("sysparm_limit", fmt.Sprintf("%d", limit))
	u.RawQuery = q.Encode()

	req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
	if err != nil {
		result.Status = StatusError
		result.Error = err.Error()
		return result
	}

	setHeaders(req)
	req.Header.Set("Accept", "application/json")
	req.Header.Set("X-UserToken", session.Token)
	for _, c := range session.Cookies {
		req.AddCookie(c)
	}

	resp, err := client.Do(req)
	if err != nil {
		result.Status = StatusError
		result.Error = err.Error()
		return result
	}
	defer resp.Body.Close()

	if resp.StatusCode == 401 || resp.StatusCode == 403 {
		result.Status = StatusNotExposed
		return result
	}
	if resp.StatusCode == 404 {
		result.Status = StatusNotExposed
		result.Error = "table not found"
		return result
	}
	if resp.StatusCode != 200 {
		result.Status = StatusError
		result.Error = fmt.Sprintf("HTTP %d", resp.StatusCode)
		return result
	}

	body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
	if err != nil {
		result.Status = StatusError
		result.Error = fmt.Sprintf("reading body: %v", err)
		return result
	}

	var apiResp tableAPIResponse
	if err := json.Unmarshal(body, &apiResp); err != nil {
		result.Status = StatusError
		result.Error = fmt.Sprintf("parsing JSON: %v", err)
		return result
	}

	if len(apiResp.Result) == 0 {
		result.Status = StatusNotExposed
		return result
	}

	// Extract field names from first record.
	fields := make([]string, 0, len(apiResp.Result[0]))
	for k := range apiResp.Result[0] {
		fields = append(fields, k)
	}
	sort.Strings(fields)

	// Extract samples.
	var samples []TableSample
	for _, record := range apiResp.Result {
		sample := TableSample{
			SysID:        tableAPIFieldString(record, "sys_id"),
			DisplayValue: tableAPIFirstField(record, "name", "number", "short_description", "title", "display_name"),
		}
		samples = append(samples, sample)
	}

	result.Status = StatusExposed
	result.RecordCount = parseXTotalCount(resp.Header.Get("X-Total-Count"), len(apiResp.Result))
	result.SampleCount = len(samples)
	result.Fields = fields
	result.Samples = samples
	result.Hints = ClassifyTable(table, "", samples)
	if len(result.Hints) == 0 {
		result.Hints = ReviewRequiredHint(result.RecordCount)
	}
	result.Reproduction = buildTableAPIReproduction(session, table, limit, resp.StatusCode)

	return result
}

// EnumerateTableAPI probes tables via the REST API with bounded concurrency.
func EnumerateTableAPI(ctx context.Context, session *Session, tables []string, limit, concurrency int, client *http.Client) []TableAPIResult {
	if concurrency < 1 {
		concurrency = 1
	}

	allResults := make([]*TableAPIResult, len(tables))
	sem := make(chan struct{}, concurrency)
	var wg sync.WaitGroup

	for i, table := range tables {
		wg.Add(1)
		go func(idx int, t string) {
			defer wg.Done()
			sem <- struct{}{}
			defer func() { <-sem }()
			allResults[idx] = ProbeTableAPI(ctx, session, t, limit, client)
		}(i, table)
	}
	wg.Wait()

	var results []TableAPIResult
	for _, r := range allResults {
		if r != nil && r.Status == StatusExposed {
			results = append(results, *r)
		}
	}
	return results
}

func buildTableAPIReproduction(session *Session, table string, limit, statusCode int) *Reproduction {
	u := url.URL{
		Scheme: "https",
		Host:   session.Target,
		Path:   "/api/now/table/" + table,
	}
	q := u.Query()
	q.Set("sysparm_limit", fmt.Sprintf("%d", limit))
	u.RawQuery = q.Encode()

	headers := map[string]string{
		"Accept":      "application/json",
		"X-UserToken": session.Token,
	}
	if cookie := cookieHeader(session.Cookies); cookie != "" {
		headers["Cookie"] = cookie
	}

	return &Reproduction{
		Method:     "GET",
		URL:        u.String(),
		Headers:    headers,
		StatusCode: statusCode,
		Curl:       curlCommand("GET", u.String(), headers, ""),
	}
}

func parseXTotalCount(header string, fallback int) int {
	if header == "" {
		return fallback
	}
	n, err := strconv.Atoi(strings.TrimSpace(header))
	if err != nil || n < 1 {
		return fallback
	}
	return n
}

func tableAPIFieldString(record map[string]any, key string) string {
	if v, ok := record[key]; ok {
		if s, ok := v.(string); ok {
			return s
		}
	}
	return ""
}

func tableAPIFirstField(record map[string]any, keys ...string) string {
	for _, key := range keys {
		if v := tableAPIFieldString(record, key); v != "" {
			return v
		}
	}
	// Fallback: first non-empty string value.
	for _, v := range record {
		if s, ok := v.(string); ok && strings.TrimSpace(s) != "" && len(s) < 100 {
			return s
		}
	}
	return ""
}