feat: Add initial parse functionality. Add stub test object.

Author: ldmi3i

go.mod

@@ -3,6 +3,7 @@ module github.com/rog-golang-buddies/api-hub_data-scraping-service
 go 1.18
 
 require (
+	github.com/getkin/kin-openapi v0.98.0
 	github.com/golang/mock v1.6.0
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/rabbitmq/amqp091-go v1.4.0
@@ -13,8 +14,13 @@ require (
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/swag v0.19.5 // indirect
+	github.com/invopop/yaml v0.1.0 // indirect
+	github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -3,15 +3,27 @@ github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/getkin/kin-openapi v0.98.0 h1:lIACvCG9cxmFsEywz+LCoVhcZHFLUy+Nv5QSkb43eAE=
+github.com/getkin/kin-openapi v0.98.0/go.mod h1:w4lRPHiyOdwGbOkLIyk+P0qCwlu7TXPCHD/64nSXzgE=
+github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
+github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
+github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY=
+github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc=
+github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q=
 github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
 github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8=
+github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -21,6 +33,7 @@ github.com/rabbitmq/amqp091-go v1.4.0/go.mod h1:JsV0ofX5f1nwOGafb8L5rBItt9GyhfQf
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
@@ -66,9 +79,12 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/app.go

@@ -7,6 +7,7 @@ import (
 	"github.com/rog-golang-buddies/api-hub_data-scraping-service/internal/load"
 	"github.com/rog-golang-buddies/api-hub_data-scraping-service/internal/logger"
 	"github.com/rog-golang-buddies/api-hub_data-scraping-service/internal/parse"
+	"github.com/rog-golang-buddies/api-hub_data-scraping-service/internal/parse/openapi"
 	"github.com/rog-golang-buddies/api-hub_data-scraping-service/internal/process"
 	"github.com/rog-golang-buddies/api-hub_data-scraping-service/internal/queue"
 	"github.com/rog-golang-buddies/api-hub_data-scraping-service/internal/queue/handler"
@@ -67,7 +68,7 @@ func Start() int {
 
 func createDefaultProcessor() (process.UrlProcessor, error) {
 	recognizer := recognize.NewRecognizer()
-	parsers := []parse.Parser{parse.NewJsonOpenApiParser(), parse.NewYamlOpenApiParser()}
+	parsers := []parse.Parser{openapi.NewJsonOpenApiParser(), openapi.NewYamlOpenApiParser()}
 	converter := parse.NewConverter(parsers)
 	loader := load.NewContentLoader()
 

internal/dto/apiSpecDoc/apiSpecDoc.go

@@ -1,9 +1,9 @@
 package apiSpecDoc
 
-//ApiSpecDoc represents full API Specification document
-//with all required data to view and execute requests
-//Propose to not store something like components at the initial stage - leave it as update to simplify logic
-//So each ApiMethod need to keep all schema (some duplication, but we could update it faster with extending of the model)
+// ApiSpecDoc represents full API Specification document
+// with all required data to view and execute requests
+// Propose to not store something like components at the initial stage - leave it as update to simplify logic
+// So each ApiMethod need to keep all schema (some duplication, but we could update it faster with extending of the model)
 type ApiSpecDoc struct {
 	//Type of API definition
 	//It's possible to be OpenApi, gRPC and others
@@ -13,15 +13,15 @@ type ApiSpecDoc struct {
 	//It relates to:
 	//Tags - Open API specs
 	//Services - RPCs
-	Groups []Group
+	Groups []*Group
 
 	//Methods at the root level without groups
-	Methods []ApiMethod
+	Methods []*ApiMethod
 }
 
-//Group represents some grouping rule
-//Tags for the open API
-//Services for the gRPC
+// Group represents some grouping rule
+// Tags for the open API
+// Services for the gRPC
 type Group struct {
 	//Name of the group
 	Name string
@@ -30,10 +30,10 @@ type Group struct {
 	Description string
 
 	//Methods is a set of request methods related to the group
-	Methods []ApiMethod
+	Methods []*ApiMethod
 }
 
-//ApiMethod represents particular API method to call
+// ApiMethod represents particular API method to call
 type ApiMethod struct {
 	//Path to send request
 	//For example /open/api/info/
@@ -52,7 +52,7 @@ type ApiMethod struct {
 	Type MethodType
 
 	//RequestBody is a description of ApiMethod request body, nil if no request body
-	RequestBody *MediaTypeObject
+	RequestBody *RequestBody
 
 	//Servers represent available paths for requests with description.
 	//In the case of swagger we need to calculate it taking the deepest nested servers definition
@@ -135,29 +135,29 @@ type Schema struct {
 
 	//Nested describe nested object/properties
 	//If it's object and contain nested fields
-	Fields []Schema
+	Fields []*Schema
 }
 
-//MediaTypeObject represents schema for the different media types
-//i.e. "application/json" and etc.
+// MediaTypeObject represents schema for the different media types
+// i.e. "application/json" and etc.
 type MediaTypeObject struct {
-	Schema Schema
+	Schema *Schema
 }
 
-//RequestBody is a representation of request body
+// RequestBody is a representation of request body
 type RequestBody struct {
 	Description string
 
 	//Content represents request object for the different media types
-	Content map[string]MediaTypeObject
+	Content map[string]*MediaTypeObject
 
 	//Required define is request body required
 	Required bool
 }
 
-//Server represents server description
-//To use in the address line of view
-//https://swagger.io/specification/#server-object
+// Server represents server description
+// To use in the address line of view
+// https://swagger.io/specification/#server-object
 type Server struct {
 	//Url to access server (no templates, real url)
 	Url string
@@ -166,8 +166,8 @@ type Server struct {
 	Description string
 }
 
-//Parameter is abstraction of additional data to request
-//It is headers for the REST API and metadata for the gRPC
+// Parameter is abstraction of additional data to request
+// It is headers for the REST API and metadata for the gRPC
 type Parameter struct {
 	Name string
 
@@ -183,9 +183,9 @@ type Parameter struct {
 	Required bool
 }
 
-//ExternalDoc may be available for the Open API
-//And contain link to description of request method
-//Maybe useful also for debugging - to find failed API description rapidly
+// ExternalDoc may be available for the Open API
+// And contain link to description of request method
+// Maybe useful also for debugging - to find failed API description rapidly
 type ExternalDoc struct {
 	Description string
 

internal/dto/apiSpecDoc/types.go

@@ -1,30 +1,58 @@
 package apiSpecDoc
 
-//Type represents type of parsed API
+// Type represents type of parsed API
 type Type int
 
 const (
 	TypeOpenApi Type = iota
 )
 
-//SchemaType is the type of field.
+// SchemaType is the type of field.
 type SchemaType string
 
 const (
-	Integer SchemaType = "INTEGER"
-	Number  SchemaType = "NUMBER"
-	String  SchemaType = "STRING"
-	Date    SchemaType = "DATE"
-	Array   SchemaType = "ARRAY"  //Array type may contain single nested field with type to define full array type
-	Map     SchemaType = "MAP"    //Map i.e. { "map_name": {"key1":"value1", "key2":"value2"}}
-	OneOf   SchemaType = "ONE_OF" //OneOf is one of the different types (C union) (from nested fields)
-	Object  SchemaType = "OBJECT" //Object represent object and contains set of fields inside
+	Unknown    SchemaType = "UNKNOWN"
+	NotDefined SchemaType = "NOT_DEFINED"
+	Integer    SchemaType = "INTEGER"
+	Boolean    SchemaType = "BOOLEAN"
+	Number     SchemaType = "NUMBER"
+	String     SchemaType = "STRING"
+	Date       SchemaType = "DATE"
+	Array      SchemaType = "ARRAY"  //Array type may contain single nested field with type to define full array type
+	Map        SchemaType = "MAP"    //Map i.e. { "map_name": {"key1":"value1", "key2":"value2"}}
+	OneOf      SchemaType = "ONE_OF" //OneOf is one of the different types (C union) (from nested fields)
+	AnyOf      SchemaType = "ANY_OF" //AnyOf defines that result object can contain any set of sub-schemes
+	AllOf      SchemaType = "ALL_OF" //AllOf defines that result object combines all listed objects/properties.
+	Not        SchemaType = "NOT"    //Not represents type that can't be used. So it's possible to use any of types except "not"
+	Object     SchemaType = "OBJECT" //Object represent object and contains set of fields inside
 )
 
-//MethodType define type of action of the method
+func ResolveSchemaType(strType string) SchemaType {
+	switch strType {
+	case "string":
+		return String
+	case "number":
+		return Number
+	case "integer":
+		return Integer
+	case "boolean":
+		return Boolean
+	case "array":
+		return Array
+	case "object":
+		return Object
+	case "":
+		return NotDefined
+	default:
+		return Unknown
+	}
+}
+
+// MethodType define type of action of the method
 type MethodType string
 
 const (
+	MethodConnect MethodType = "CONNECT"
 	MethodGet     MethodType = "GET"
 	MethodPut     MethodType = "PUT"
 	MethodPost    MethodType = "POST"
@@ -36,7 +64,7 @@ const (
 	//Unary/bidirectional stream etc. for the rpc
 )
 
-//ParameterType represents to what part of request parameter relates
+// ParameterType represents to what part of request parameter relates
 type ParameterType string
 
 const (

internal/parse/converter.go

@@ -26,7 +26,7 @@ func (c *ConverterImpl) Convert(file *fileresource.FileResource) (*apiSpecDoc.Ap
 	if !ok {
 		return nil, errors.New("file type not supported")
 	}
-	apiSpec, err := parser.parse(file.Content)
+	apiSpec, err := parser.Parse(file.Content)
 	if err != nil {
 		return nil, err
 	}
@@ -37,7 +37,7 @@ func (c *ConverterImpl) Convert(file *fileresource.FileResource) (*apiSpecDoc.Ap
 func NewConverter(parsers []Parser) Converter {
 	parsersMap := make(map[fileresource.AsdFileType]Parser)
 	for _, parser := range parsers {
-		parsersMap[parser.getType()] = parser
+		parsersMap[parser.GetType()] = parser
 	}
 	return &ConverterImpl{
 		parsers: parsersMap,

internal/parse/converter_test.go

@@ -1,12 +1,13 @@
 package parse
 
 import (
+	"github.com/rog-golang-buddies/api-hub_data-scraping-service/internal/parse/openapi"
 	"github.com/stretchr/testify/assert"
 	"testing"
 )
 
 func TestNewConverter(t *testing.T) {
-	parsers := []Parser{NewYamlOpenApiParser(), NewJsonOpenApiParser()}
+	parsers := []Parser{openapi.NewYamlOpenApiParser(), openapi.NewJsonOpenApiParser()}
 	converter := NewConverter(parsers)
 	assert.NotNil(t, converter)
 }

internal/parse/jsonOpenApiParser.go renamed to internal/parse/openapi/jsonOpenApiParser.go

@@ -1,23 +1,24 @@
-package parse
+package openapi
 
 import (
 	"errors"
 	"github.com/rog-golang-buddies/api-hub_data-scraping-service/internal/dto/apiSpecDoc"
 	"github.com/rog-golang-buddies/api-hub_data-scraping-service/internal/dto/fileresource"
+	"github.com/rog-golang-buddies/api-hub_data-scraping-service/internal/parse"
 )
 
-//JsonOpenApiParser implementation for parsing json open API files
+// JsonOpenApiParser implementation for parsing json open API files
 type JsonOpenApiParser struct {
 }
 
-func (joap *JsonOpenApiParser) parse(content []byte) (*apiSpecDoc.ApiSpecDoc, error) {
+func (joap *JsonOpenApiParser) Parse(content []byte) (*apiSpecDoc.ApiSpecDoc, error) {
 	return nil, errors.New("not implemented")
 }
 
-func (joap *JsonOpenApiParser) getType() fileresource.AsdFileType {
+func (joap *JsonOpenApiParser) GetType() fileresource.AsdFileType {
 	return fileresource.JsonOpenAPI
 }
 
-func NewJsonOpenApiParser() Parser {
+func NewJsonOpenApiParser() parse.Parser {
 	return &JsonOpenApiParser{}
 }