better-analysis

Summary:

- Strategy for `aws` `v4` signing in comments.
- Strategy for namespace and aliasing.
- Aliasing ideas expanding.
- Beginnings of address space analysis.
- Unit tests working.
- Canoncial test coverage.
- Faster analysis.

Author: general-kroll-4-life

anysdk/addressable.go

@@ -19,6 +19,13 @@ func (ns *namedSchema) GetName() string {
 	return ns.name
 }
 
+func (ns *namedSchema) GetAlias() string {
+	if ns.s != nil {
+		return ns.s.GetAlias()
+	}
+	return ""
+}
+
 func (ns *namedSchema) GetSchema() (Schema, bool) {
 	return ns.s, true
 }
@@ -65,6 +72,7 @@ type Addressable interface {
 	ConditionIsValid(lhs string, rhs interface{}) bool
 	GetLocation() string
 	GetName() string
+	GetAlias() string
 	GetSchema() (Schema, bool)
 	GetType() string
 	IsRequired() bool

anysdk/const.go

@@ -21,6 +21,7 @@ const (
 	ExtensionKeyProvider       string = "x-stackql-provider"
 	ExtensionKeyResources      string = "x-stackQL-resources"
 	ExtensionKeyStringOnly     string = "x-stackQL-stringOnly"
+	ExtensionKeyAlias          string = "x-stackQL-alias"
 )
 
 const (

anysdk/loader.go

@@ -317,7 +317,7 @@ func (l *standardLoader) mergeResources(svc OpenAPIService, rscMap map[string]Re
 		if rsc.GetServiceDocPath() != nil {
 			sr = rsc.GetServiceDocPath()
 		}
-		err := l.mergeResource(svc, rsc, sr)
+		err := l.mergeResource(k, svc, rsc, sr)
 		if err != nil {
 			return err
 		}
@@ -330,7 +330,7 @@ func (l *standardLoader) mergeResourcesScoped(svc OpenAPIService, svcUrl string,
 	scopedMap := make(map[string]Resource)
 	for k, rsc := range rr.GetResources() {
 		if rr.ObtainServiceDocUrl(k) == svcUrl {
-			err := l.mergeResource(svc, rsc, &ServiceRef{Ref: svcUrl})
+			err := l.mergeResource(k, svc, rsc, &ServiceRef{Ref: svcUrl})
 			if err != nil {
 				return err
 			}
@@ -349,14 +349,28 @@ func (l *standardLoader) mergeResourcesScoped(svc OpenAPIService, svcUrl string,
 	return nil
 }
 
-func (l *standardLoader) mergeResource(svc OpenAPIService,
+func (l *standardLoader) mergeResource(
+	rscKey string,
+	svc OpenAPIService,
 	rsc Resource,
 	sr *ServiceRef,
 ) error {
 	rsc.setService(svc) // must happen before resolving inverses
+	existingMethods := make(Methods)
+	existingResource, existingResourceErr := svc.GetResource(rscKey)
+	if existingResourceErr == nil && existingResource != nil {
+		existingMethods = existingResource.GetMethods()
+	}
 	for k, vOp := range rsc.GetMethods() {
 		v := vOp
 		v.setMethodKey(k)
+		existingMethod, existingMethodExists := existingMethods[k]
+		if existingMethodExists {
+			v = existingMethod
+			v.setMethodKey(k)
+			v.setResource(rsc)
+			continue
+		}
 		// TODO: replicate this for the damned inverse
 		err := l.resolveOperationRef(svc, rsc, &v, v.GetPathRef(), sr)
 		if err != nil {
@@ -763,6 +777,13 @@ func (loader *standardLoader) resolveOperationRef(doc OpenAPIService, rsc Resour
 	}
 
 	component.setOperationRef(&OperationRef{Value: op, Ref: component.GetOperationRef().Ref})
+	response, responseExists := component.GetResponse()
+	if responseExists {
+		err = loader.resolveExpectedResponse(doc, component.GetOperationRef().Value, response)
+		if err != nil {
+			return err
+		}
+	}
 	component.setPathItem(pi)
 	err = loader.extractAndMergeQueryTransposeOpLevel(component)
 	if err != nil {

anysdk/operation_store.go

@@ -110,6 +110,11 @@ type OperationStore interface {
 
 type StandardOperationStore interface {
 	OperationStore
+	// Assist analysis
+	GetRequestBodyAttributesNoRename() (map[string]Addressable, error)
+	GetSchemaAtPath(key string) (Schema, error)
+	GetSelectItemsKeySimple() string
+	LookupSelectItemsKey() string
 	//
 	getQueryTransposeAlgorithm() string
 	getRequiredNonBodyParameters() map[string]Addressable
@@ -624,6 +629,10 @@ func (m *standardOpenAPIOperationStore) getUnionRequiredParameters() (map[string
 	return m.Resource.getUnionRequiredParameters(m)
 }
 
+func (m *standardOpenAPIOperationStore) GetSelectItemsKeySimple() string {
+	return m.getSelectItemsKeySimple()
+}
+
 func (m *standardOpenAPIOperationStore) getSelectItemsKeySimple() string {
 	if m.Response != nil {
 		return m.Response.ObjectKey
@@ -1048,6 +1057,10 @@ func (m *standardOpenAPIOperationStore) getName() string {
 	return m.MethodKey
 }
 
+func (m *standardOpenAPIOperationStore) GetRequestBodyAttributesNoRename() (map[string]Addressable, error) {
+	return m.getRequestBodyAttributesNoRename()
+}
+
 func (m *standardOpenAPIOperationStore) ToPresentationMap(extended bool) map[string]interface{} {
 	requiredParams := m.getRequiredNonBodyParameters()
 	var requiredParamNames []string
@@ -1415,6 +1428,19 @@ func (op *standardOpenAPIOperationStore) GetSelectSchemaAndObjectPath() (Schema,
 	return nil, "", fmt.Errorf("no response body for operation =  %s", op.GetName())
 }
 
+func (op *standardOpenAPIOperationStore) GetSchemaAtPath(path string) (Schema, error) {
+	k := path
+	if op.Response != nil && op.Response.OverrideSchema != nil && op.Response.OverrideSchema.Value != nil {
+		rv, _, err := op.Response.OverrideSchema.Value.getSelectItemsSchema(k, op.Response.OverrideBodyMediaType)
+		return rv, err
+	}
+	if op.Response != nil && op.Response.Schema != nil {
+		rv, _, err := op.Response.Schema.getSelectItemsSchema(k, op.getOptimalResponseMediaType())
+		return rv, err
+	}
+	return nil, fmt.Errorf("no response body for operation =  %s", op.GetName())
+}
+
 type ProcessedOperationResponse interface {
 	GetResponse() (response.Response, bool)
 	GetReversal() (HTTPPreparator, bool)
@@ -1551,6 +1577,10 @@ func (op *standardOpenAPIOperationStore) ProcessResponse(httpResponse *http.Resp
 	return newStandardOperationResponse(rv, reversal), err
 }
 
+func (ops *standardOpenAPIOperationStore) LookupSelectItemsKey() string {
+	return ops.lookupSelectItemsKey()
+}
+
 func (ops *standardOpenAPIOperationStore) lookupSelectItemsKey() string {
 	s := ops.getSelectItemsKeySimple()
 	if s != "" {

anysdk/params.go

@@ -42,6 +42,10 @@ func (p *standardParameter) GetName() string {
 	return p.Name
 }
 
+func (p *standardParameter) GetAlias() string {
+	return getAliasFromExtensions(p.ExtensionProps)
+}
+
 func (p *standardParameter) GetLocation() string {
 	return p.In
 }
@@ -72,6 +76,11 @@ func (p *standardParameter) GetType() string {
 func (p parameters) getParameterFromInSubset(key, inSubset string) (Addressable, bool) {
 	for _, paramRef := range p.Parameters {
 		param := paramRef.Value
+		paramAlias := getAliasFromExtensions(param.ExtensionProps)
+		hasAlias := paramAlias != ""
+		if hasAlias && param.In == inSubset && paramAlias == key {
+			return NewParameter(param, p.svc), true
+		}
 		if param.In == inSubset && param.Name == key {
 			return NewParameter(param, p.svc), true
 		}

anysdk/resource.go

@@ -31,13 +31,17 @@ type Resource interface {
 	GetFirstMethodFromSQLVerb(sqlVerb string) (StandardOperationStore, string, bool)
 	GetFirstMethodMatchFromSQLVerb(sqlVerb string, parameters map[string]interface{}) (StandardOperationStore, map[string]interface{}, bool)
 	GetService() (OpenAPIService, bool)
+	GetProvider() (Provider, bool)
 	GetViewsForSqlDialect(sqlDialect string) ([]View, bool)
 	GetMethodsMatched() Methods
 	ToMap(extended bool) map[string]interface{}
 	// unexported mutators
 	getSQLVerbs() map[string][]OpenAPIOperationStoreRef
 	setProvider(p Provider)
 	setService(s OpenAPIService)
+	SetProvider(p Provider)
+	SetService(s OpenAPIService)
+	SetProviderService(ps ProviderService)
 	setProviderService(ps ProviderService)
 	getUnionRequiredParameters(method StandardOperationStore) (map[string]Addressable, error)
 	setMethod(string, *standardOpenAPIOperationStore)
@@ -83,10 +87,21 @@ func (r *standardResource) GetService() (OpenAPIService, bool) {
 	return r.OpenAPIService, true
 }
 
+func (r *standardResource) GetProvider() (Provider, bool) {
+	if r.Provider == nil {
+		return nil, false
+	}
+	return r.Provider, true
+}
+
 func (r *standardResource) getSQLVerbs() map[string][]OpenAPIOperationStoreRef {
 	return r.SQLVerbs
 }
 
+func (r *standardResource) SetService(s OpenAPIService) {
+	r.setService(s)
+}
+
 func (r *standardResource) setService(s OpenAPIService) {
 	r.OpenAPIService = s
 }
@@ -105,6 +120,14 @@ func (r *standardResource) setMethod(k string, v *standardOpenAPIOperationStore)
 	r.Methods[k] = *v
 }
 
+func (r *standardResource) SetProvider(p Provider) {
+	r.Provider = p
+}
+
+func (r *standardResource) SetProviderService(ps ProviderService) {
+	r.ProviderService = ps
+}
+
 func (r *standardResource) setProvider(p Provider) {
 	r.Provider = p
 }

anysdk/schema.go

@@ -47,6 +47,7 @@ type Schema interface {
 	GetType() string
 	GetPropertySchema(key string) (Schema, error)
 	GetRequired() []string
+	GetAlias() string
 	GetSelectSchema(itemsKey, mediaType string) (Schema, string, error)
 	IsArrayRef() bool
 	IsBoolean() bool
@@ -135,6 +136,17 @@ func (s *standardSchema) getAdditionalProperties() (Schema, bool) {
 	return nil, false
 }
 
+func (s *standardSchema) GetAlias() string {
+	alias, hasAlias := s.Extensions[ExtensionKeyAlias]
+	if hasAlias {
+		aliasStr, isStr := alias.(string)
+		if isStr {
+			return aliasStr
+		}
+	}
+	return ""
+}
+
 func (s *standardSchema) setPropertyOpenapi3(k string, ps *openapi3.SchemaRef) {
 	s.Properties[k] = ps
 }
@@ -787,6 +799,11 @@ func (schema *standardSchema) extractMediaTypeSynonym(mediaType string) string {
 	return mediaType
 }
 
+func (schema *standardSchema) GetSchemaAtPath(key string, mediaType string) (Schema, error) {
+	rv, _, err := schema.getSelectItemsSchema(key, mediaType)
+	return rv, err
+}
+
 func (schema *standardSchema) getSelectItemsSchema(key string, mediaType string) (Schema, string, error) {
 	if key == "" {
 		if schema.Items != nil && schema.Items.Value != nil {

anysdk/shims.go

@@ -6,6 +6,7 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/stackql/any-sdk/pkg/brickmap"
 	"github.com/stackql/stackql-parser/go/vt/sqlparser"
 )
@@ -289,3 +290,14 @@ func getStringFromStringFunc(fe *sqlparser.FuncExpr) (string, error) {
 	}
 	return "", fmt.Errorf("cannot extract string from func '%s'", fe.Name)
 }
+
+func getAliasFromExtensions(extensionObj openapi3.ExtensionProps) string {
+	alias, hasAlias := extensionObj.Extensions[ExtensionKeyAlias]
+	if hasAlias {
+		aliasStr, isStr := alias.(string)
+		if isStr {
+			return aliasStr
+		}
+	}
+	return ""
+}

docs/aliasing.md

@@ -0,0 +1,74 @@
+
+
+# Aliasing
+
+## In the abstract
+
+
+Any query's address space is composed of:
+
+1. Zero or one URLs (eg https://host:port/path/to/resource?queryParam1=a&queryParam2=b).
+2. Input data on some logical "page" eg: an HTTP request.  Another example: a golang template and fuinctionality to call a CLI tool.
+3. Returned data on some logical "page", eg: an HTTP response.
+
+Pages can in theory be collections of pages and populated with a mixture of static and dynamic data.  The namespace is a search structure to locate any address.  
+
+> For arbitrary address location, some system that traverses the entire address space at arbitrary depth.
+
+`openapi` provides some, but ceertainly not all, of this functionality.    Putting aside the implementation, this is possible so long as page collections are named or strictly ordered.  
+
+### Desired Namespace
+
+For SQL semantics, a flat namespace without arbitrary prefixes and some semantic relevance is preferred.  This suggests a configurable aliasing capability, coupled with a collision resolution algorithm.  The aliasing capability:
+
+- For input data, must support transparent rewrite w.r.t. the provider system.  Ie: the provider receives precisely the unaliased data.  This implies that any existing rewrite logic must either be ignored or be consistent with the aliasing.
+- For output data, simply a lazy rewrite before staging in RDBMS / relational algebra engine is ok.
+
+Something like:
+
+- Flatten the address space based upon config and default behaviour.  There is already a simply version of this in `objectKey`.
+- Search the flattened address space based on cofiguration and default behaviour, identifying conflicts.
+- For each conflict, apply a configurable resolution algorithm.
+- At runtime, perform the aliasing transform relations and display data accordingly.
+
+
+
+## For HTTP
+
+
+A useful consideration to begin with is the native capabilities of `openapi`, which supports dynamic substitution of URLs and requests for:
+
+- Named [`parameter` objects](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.0.md#parameter-object) in "query", "header", "path" or "cookie" aspects.  These are unique in the tuple (`name`, `location`), and map to one or more locations in the query address space.  For an example of plurality: query parameters may be repeated per [RFC 3986](https://datatracker.ietf.org/doc/html/rfc3986).
+- Named [server `variable` objects](https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.0.0.md#server-variable-object) in server URLs.
+
+
+Request and response **sub-component** (body, header) attributes can be exposed unqualified, provided they do not collide with other exposed attributes.  Such collisions are frequent and so naive and sub-optimal measures are available:
+
+- Not exposing response header attributes at all.
+- Heuristics that effectively clobber any input attribute that collides with a server variable.
+- Routing request body attributes based on a prefix eg: `data__`.  This is undesirable on the grounds of poor encapsulation and weakened semantics.
+
+### Proposed HTTP implementation
+
+Requirements:
+
+- Aliased `openapi` parameters to include an alias extension attribute.
+- Some config exists denoting aliased request and response page attributes.  Call this `PageAliasDirectory`.
+- **v2** The existing `objectKey` is enhanced / replaced with a fucnction that supports unions etc.
+- A flattening algorithm exists.  Eg: strings -> string, int -> int, object -> string....
+
+Then:
+
+- For all `openapi` parameters, cache alias extension attribute.
+- AOT validate flattened namespace.
+    - Build namespace search structure.  This will both detect violations and be used downstream, for search and rewrite.
+- Runtime perform transform relations.
+
+Search structure:
+
+- Want to be extensible to arbitrary depth.
+   - This suggests tree / graph rather than flat map.
+   - Some kind of Trie (prefix tree).
+- Sits at method level.
+   - Makes sense to place aliasing at sql method level, eg sql 
+

pkg/awssign/aws_sign.go

@@ -15,6 +15,16 @@ import (
 	"github.com/aws/aws-sdk-go-v2/credentials"
 )
 
+// This interface is not fully compliant.
+// Ultimately, for full coverage,
+// we need to emulate [the SDK auth specifications](https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-auth.html).
+//
+// This is the sort of stuff we need to emulate:
+//   - [Resolver doc based resolution](https://github.com/aws/aws-sdk-go-v2/blob/2e08461090ccba679456c05264e2c04bf228138e/service/accessanalyzer/options.go#L150).
+//   - SDK doc based code gen settings for auth:
+//       - [For the `account` service](https://github.com/aws/aws-sdk-go-v2/blob/3ac24f20bb3b05955fcb1b3fae7883d3a03fe60d/codegen/sdk-codegen/aws-models/account.json#L133).
+//
+
 var (
 	_                Transport = &standardAwsSignTransport{}
 	emptyPayloadHash string    = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"