Skip to content

Commit 40358ee

Browse files
wajdawajda
committed
Merge pull request #25 in BACIBBD/spline from feature/SL-61 to feature/SL-43-new-data-model
* commit '990b5674374999f8e2c8fff8c7c7a319f4f73fc8': SL-61 Angular/material: Replace 'MD' suffixes to 'MAT' according to the recent change in the library (see angular/components#7241) SL-61 Redo graph visualization + fix types + fix server side unit tests
2 parents 3545f3c + 990b567 commit 40358ee

File tree

60 files changed

+2039
-1526
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+2039
-1526
lines changed

‎model/src/main/scala/za/co/absa/spline/model/PersistedDatasetDescriptor.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
package za.co.absa.spline.model
1818

19-
import java.net.URL
19+
import java.net.URI
2020
import java.util.UUID
2121

2222
/**
@@ -35,6 +35,6 @@ case class PersistedDatasetDescriptor
3535
appId: String,
3636
appName: String,
3737
lineageId: UUID,
38-
path: URL,
38+
path: URI,
3939
timestamp: Long
4040
)

‎model/src/main/scala/za/co/absa/spline/model/expr/Expression.scala

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ sealed trait Expression {
5656
* The case class represents Spark expressions for which a dedicated expression node type hasn't been created yet.
5757
*
5858
* @param exprType see [[za.co.absa.spline.model.expr.Expression#exprType Expression.exprType]]
59-
* @param text see [[za.co.absa.spline.model.expr.Expression#textualRepresentation Expression.textualRepresentation]]
59+
* @param text see [[za.co.absa.spline.model.expr.Expression#text Expression.text]]
6060
* @param dataType see [[za.co.absa.spline.model.expr.Expression#dataType Expression.dataType]]
6161
* @param children see [[za.co.absa.spline.model.expr.Expression#children Expression.children]]
6262
*/
@@ -72,7 +72,7 @@ case class Generic
7272
* The case class represents renaming of an underlying expression to a specific alias.
7373
*
7474
* @param alias A final name of the expression
75-
* @param text see [[za.co.absa.spline.model.expr.Expression#textualRepresentation Expression.textualRepresentation]]
75+
* @param text see [[za.co.absa.spline.model.expr.Expression#text Expression.text]]
7676
* @param dataType see [[za.co.absa.spline.model.expr.Expression#dataType Expression.dataType]]
7777
* @param children see [[za.co.absa.spline.model.expr.Expression#children Expression.children]]
7878
*/
@@ -94,7 +94,7 @@ case class Alias
9494
*
9595
* @param exprType see [[za.co.absa.spline.model.expr.Expression#exprType Expression.exprType]]
9696
* @param symbol A symbol expressing the operation (+, -, *, /, etc. )
97-
* @param text see [[za.co.absa.spline.model.expr.Expression#textualRepresentation Expression.textualRepresentation]]
97+
* @param text see [[za.co.absa.spline.model.expr.Expression#text Expression.text]]
9898
* @param dataType see [[za.co.absa.spline.model.expr.Expression#dataType Expression.dataType]]
9999
* @param children see [[za.co.absa.spline.model.expr.Expression#children Expression.children]]
100100
*/
@@ -110,7 +110,7 @@ case class Binary
110110
/**
111111
* The case class represents a special expression for removing an attribute from a data set.
112112
*
113-
* @param text see [[za.co.absa.spline.model.expr.Expression#textualRepresentation Expression.textualRepresentation]]
113+
* @param text see [[za.co.absa.spline.model.expr.Expression#text Expression.text]]
114114
* @param dataType see [[za.co.absa.spline.model.expr.Expression#dataType Expression.dataType]]
115115
* @param children see [[za.co.absa.spline.model.expr.Expression#children Expression.children]]
116116
*/
@@ -145,7 +145,7 @@ object AttributeRemoval {
145145
*
146146
* @param refId An unique of a referenced attribute
147147
* @param name A name of a referenced attribute
148-
* @param text see [[za.co.absa.spline.model.expr.Expression#textualRepresentation Expression.textualRepresentation]]
148+
* @param text see [[za.co.absa.spline.model.expr.Expression#text Expression.text]]
149149
* @param dataType see [[za.co.absa.spline.model.expr.Expression#dataType Expression.dataType]]
150150
*/
151151
case class AttributeReference
@@ -194,7 +194,7 @@ object AttributeReference {
194194
* The case class represents a special expression describing an user-defined function of Spark.
195195
*
196196
* @param name A name assigned to an user-defined function
197-
* @param text see [[za.co.absa.spline.model.expr.Expression#textualRepresentation Expression.textualRepresentation]]
197+
* @param text see [[za.co.absa.spline.model.expr.Expression#text Expression.text]]
198198
* @param dataType see [[za.co.absa.spline.model.expr.Expression#dataType Expression.dataType]]
199199
* @param children see [[za.co.absa.spline.model.expr.Expression#children Expression.children]]
200200
*/

‎persistence/api/src/main/scala/za/co/absa/spline/persistence/api/serialization/BSONSalatContext.scala

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,5 @@ object BSONSalatContext {
2828
override val name: String = "BSON Salat Context"
2929

3030
registerGlobalKeyOverride("id", "_id")
31-
registerGlobalKeyOverride("dataLineageID", "_id")
3231
}
3332
}

‎persistence/atlas/src/main/atlas/spline-meta-model.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -387,7 +387,7 @@
387387
"isUnique": false
388388
},
389389
{
390-
"name": "textualRepresentation",
390+
"name": "text",
391391
"typeName": "string",
392392
"cardinality": "SINGLE",
393393
"isIndexable": false,

‎persistence/atlas/src/main/scala/za/co/absa/spline/persistence/atlas/model/Expression.scala

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,15 +27,15 @@ import scala.collection.JavaConverters._
2727
/**
2828
* The case case represents common properties for all expression types
2929
* @param qualifiedName An unique expression
30-
* @param textualRepresentation A textual representation of the expression
30+
* @param text A textual representation of the expression
3131
* @param expressionType A type of the expression
3232
* @param dataType A data type associated with the expression
3333
* @param children A sequence of sub-expressions
3434
*/
3535
case class ExpressionCommonProperties
3636
(
3737
qualifiedName : String,
38-
textualRepresentation: String,
38+
text: String,
3939
expressionType: String,
4040
dataType: DataType,
4141
children: Seq[Expression]
@@ -54,9 +54,9 @@ class Expression(
5454
) extends Referenceable (
5555
entityType,
5656
new java.util.HashMap[String, Object]{
57-
put(AtlasClient.NAME, commonProperties.textualRepresentation)
57+
put(AtlasClient.NAME, commonProperties.text)
5858
put(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, commonProperties.qualifiedName)
59-
put("textualRepresentation", commonProperties.textualRepresentation)
59+
put("text", commonProperties.text)
6060
put("expressionType", commonProperties.expressionType)
6161
put("dataType", commonProperties.dataType)
6262
put("children", commonProperties.children.asJava)

‎persistence/mongo/src/main/scala/za/co/absa/spline/persistence/mongo/MongoDataLineageReader.scala

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,16 @@
1616

1717
package za.co.absa.spline.persistence.mongo
1818

19+
import java.util.Arrays.asList
1920
import java.util.UUID
2021

21-
import com.mongodb.casbah.Imports._
2222
import _root_.salat._
23+
import com.mongodb.casbah.Imports._
24+
import za.co.absa.spline.common.FutureImplicits._
2325
import za.co.absa.spline.model.{DataLineage, PersistedDatasetDescriptor}
2426
import za.co.absa.spline.persistence.api.DataLineageReader
25-
import scala.collection.JavaConverters._
26-
import za.co.absa.spline.common.FutureImplicits._
2727

28+
import scala.collection.JavaConverters._
2829
import scala.concurrent.Future
2930

3031
/**
@@ -52,9 +53,24 @@ class MongoDataLineageReader(connection: MongoConnection) extends DataLineageRea
5253
* @return Descriptors of all data lineages
5354
*/
5455
override def list(): Future[Iterator[PersistedDatasetDescriptor]] = Future {
56+
val caseClassFields = classOf[PersistedDatasetDescriptor].getDeclaredFields map (_.getName)
57+
val auxiliaryFields = Array("_ver")
58+
val fieldsToFetch = caseClassFields ++ auxiliaryFields map (_ -> 1)
59+
5560
connection.dataLineageCollection
56-
.find(DBObject(), DBObject("_id" -> 1, "_ver" -> 1, "appId" -> 1, "appName" -> 1, "timestamp" -> 1))
57-
.iterator.asScala
61+
.aggregate(asList(
62+
DBObject("$addFields" → DBObject(
63+
"___rootDS" → DBObject("$arrayElemAt" → Array("$datasets", 0)),
64+
"___rootOP" → DBObject("$arrayElemAt" → Array("$operations", 0))
65+
)),
66+
DBObject("$addFields" → DBObject(
67+
"lineageId" → "$_id",
68+
"datasetId" → "$___rootDS._id",
69+
"path" → "$___rootOP.path"
70+
)),
71+
DBObject("$project" → DBObject(fieldsToFetch: _*))
72+
))
73+
.results.iterator.asScala
5874
.map(withVersionCheck(grater[PersistedDatasetDescriptor].asObject(_)))
5975
}
6076

‎persistence/mongo/src/test/scala/za/co/absa/spline/persistence/mongo/MongoDataLineagePersistenceSpecBase.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import java.util.UUID.randomUUID
2020

2121
import org.scalatest.{AsyncFlatSpec, BeforeAndAfterEach, Matchers}
2222
import za.co.absa.spline.model.dt.Simple
23-
import za.co.absa.spline.model.op.{Generic, OperationProps}
23+
import za.co.absa.spline.model.op.{Destination, Generic, OperationProps}
2424
import za.co.absa.spline.model.{Attribute, Schema, _}
2525

2626
abstract class MongoDataLineagePersistenceSpecBase extends AsyncFlatSpec with Matchers with BeforeAndAfterEach {
@@ -49,6 +49,7 @@ abstract class MongoDataLineagePersistenceSpecBase extends AsyncFlatSpec with Ma
4949
appName,
5050
123L,
5151
Seq(
52+
Destination(OperationProps(randomUUID, "Destination", Seq(md1.id), md1.id), "parquet", "hdfs://foo/bar/path"),
5253
Generic(OperationProps(randomUUID, "Union", Seq(md1.id, md2.id), md3.id), "rawString1"),
5354
Generic(OperationProps(randomUUID, "Filter", Seq(md4.id), md2.id), "rawString2"),
5455
Generic(OperationProps(randomUUID, "LogicalRDD", Seq.empty, md4.id), "rawString3"),

‎persistence/mongo/src/test/scala/za/co/absa/spline/persistence/mongo/MongoDataLineageReaderSpec.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
package za.co.absa.spline.persistence.mongo
1818

19-
import java.net.URL
19+
import java.net.URI
2020

2121
import za.co.absa.spline.model.PersistedDatasetDescriptor
2222
import za.co.absa.spline.model.op.Destination
@@ -35,7 +35,7 @@ class MongoDataLineageReaderSpec extends MongoDataLineagePersistenceSpecBase{
3535
appId = l.appId,
3636
appName = l.appName,
3737
lineageId = l.id,
38-
path = new URL(l.rootNode.asInstanceOf[Destination].path),
38+
path = new URI(l.rootNode.asInstanceOf[Destination].path),
3939
timestamp = l.timestamp))
4040

4141
val descriptions = Future.sequence(testLineages.map(i => mongoWriter.store(i))).flatMap(_ => mongoReader.list().map(_.toSeq))

‎web/pom.xml

Lines changed: 107 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -103,46 +103,119 @@
103103
<version>1.5.0</version>
104104
<executions>
105105
<execution>
106+
<id>gen-ts-model-lineage</id>
106107
<phase>process-classes</phase>
107108
<goals>
108109
<goal>java</goal>
109110
</goals>
111+
<configuration>
112+
<includeProjectDependencies>true</includeProjectDependencies>
113+
<includePluginDependencies>true</includePluginDependencies>
114+
<executableDependency>
115+
<groupId>com.github.wajda</groupId>
116+
<artifactId>scala-ts_${scala.compat.version}</artifactId>
117+
</executableDependency>
118+
<mainClass>com.mpc.scalats.CLI</mainClass>
119+
<arguments>
120+
<argument>--out</argument>
121+
<argument>${project.basedir}/ui/src/generated-ts/lineage-model.ts</argument>
122+
<argument>--emit-interfaces</argument>
123+
<argument>--option-to-nullable</argument>
124+
<argument>za.co.absa.spline.model.PersistedDatasetDescriptor</argument>
125+
<argument>za.co.absa.spline.model.DataLineage</argument>
126+
<!-- workaround for a scala-ts bug -->
127+
<argument>za.co.absa.spline.model.op.OperationProps</argument>
128+
</arguments>
129+
</configuration>
130+
</execution>
131+
<execution>
132+
<id>gen-ts-model-operation</id>
133+
<phase>process-classes</phase>
134+
<goals>
135+
<goal>java</goal>
136+
</goals>
137+
<configuration>
138+
<includeProjectDependencies>true</includeProjectDependencies>
139+
<includePluginDependencies>true</includePluginDependencies>
140+
<executableDependency>
141+
<groupId>com.github.wajda</groupId>
142+
<artifactId>scala-ts_${scala.compat.version}</artifactId>
143+
</executableDependency>
144+
<mainClass>com.mpc.scalats.CLI</mainClass>
145+
<arguments>
146+
<argument>--out</argument>
147+
<argument>${project.basedir}/ui/src/generated-ts/operation-model.ts</argument>
148+
<argument>--emit-interfaces</argument>
149+
<argument>--option-to-nullable</argument>
150+
<argument>za.co.absa.spline.model.op.Operation</argument>
151+
<argument>za.co.absa.spline.model.op.Alias</argument>
152+
<argument>za.co.absa.spline.model.op.Destination</argument>
153+
<argument>za.co.absa.spline.model.op.Filter</argument>
154+
<argument>za.co.absa.spline.model.op.Generic</argument>
155+
<argument>za.co.absa.spline.model.op.Join</argument>
156+
<argument>za.co.absa.spline.model.op.Projection</argument>
157+
<argument>za.co.absa.spline.model.op.Source</argument>
158+
<!-- workaround for a scala-ts bug -->
159+
<argument>za.co.absa.spline.model.dt.DataType</argument>
160+
</arguments>
161+
</configuration>
162+
</execution>
163+
<execution>
164+
<id>gen-ts-model-expression</id>
165+
<phase>process-classes</phase>
166+
<goals>
167+
<goal>java</goal>
168+
</goals>
169+
<configuration>
170+
<includeProjectDependencies>true</includeProjectDependencies>
171+
<includePluginDependencies>true</includePluginDependencies>
172+
<executableDependency>
173+
<groupId>com.github.wajda</groupId>
174+
<artifactId>scala-ts_${scala.compat.version}</artifactId>
175+
</executableDependency>
176+
<mainClass>com.mpc.scalats.CLI</mainClass>
177+
<arguments>
178+
<argument>--out</argument>
179+
<argument>${project.basedir}/ui/src/generated-ts/expression-model.ts</argument>
180+
<argument>--emit-interfaces</argument>
181+
<argument>--option-to-nullable</argument>
182+
<argument>za.co.absa.spline.model.expr.Expression</argument>
183+
<argument>za.co.absa.spline.model.expr.AttributeRemoval</argument>
184+
<argument>za.co.absa.spline.model.expr.AttributeReference</argument>
185+
<argument>za.co.absa.spline.model.expr.Generic</argument>
186+
<argument>za.co.absa.spline.model.expr.Alias</argument>
187+
<argument>za.co.absa.spline.model.expr.Binary</argument>
188+
<argument>za.co.absa.spline.model.expr.UserDefinedFunction</argument>
189+
</arguments>
190+
</configuration>
191+
</execution>
192+
<execution>
193+
<id>gen-ts-model-datatype</id>
194+
<phase>process-classes</phase>
195+
<goals>
196+
<goal>java</goal>
197+
</goals>
198+
<configuration>
199+
<includeProjectDependencies>true</includeProjectDependencies>
200+
<includePluginDependencies>true</includePluginDependencies>
201+
<executableDependency>
202+
<groupId>com.github.wajda</groupId>
203+
<artifactId>scala-ts_${scala.compat.version}</artifactId>
204+
</executableDependency>
205+
<mainClass>com.mpc.scalats.CLI</mainClass>
206+
<arguments>
207+
<argument>--out</argument>
208+
<argument>${project.basedir}/ui/src/generated-ts/datatype-model.ts</argument>
209+
<argument>--emit-interfaces</argument>
210+
<argument>--option-to-nullable</argument>
211+
<argument>za.co.absa.spline.model.dt.DataType</argument>
212+
<argument>za.co.absa.spline.model.dt.Simple</argument>
213+
<argument>za.co.absa.spline.model.dt.Struct</argument>
214+
<argument>za.co.absa.spline.model.dt.Array</argument>
215+
</arguments>
216+
</configuration>
110217
</execution>
111218
</executions>
112-
<configuration>
113-
<includeProjectDependencies>true</includeProjectDependencies>
114-
<includePluginDependencies>true</includePluginDependencies>
115-
<executableDependency>
116-
<groupId>com.github.wajda</groupId>
117-
<artifactId>scala-ts_${scala.compat.version}</artifactId>
118-
</executableDependency>
119-
<mainClass>com.mpc.scalats.CLI</mainClass>
120-
<arguments>
121-
<argument>--out</argument>
122-
<argument>${project.basedir}/ui/src/generated-ts/lineage-model.ts</argument>
123-
<argument>--emit-interfaces</argument>
124-
<argument>--option-to-nullable</argument>
125-
<argument>za.co.absa.spline.model.PersistedDatasetDescriptor</argument>
126-
<argument>za.co.absa.spline.model.DataLineage</argument>
127-
<argument>za.co.absa.spline.model.expr.AttributeRemoval</argument>
128-
<argument>za.co.absa.spline.model.expr.AttrRef</argument>
129-
<argument>za.co.absa.spline.model.expr.Generic</argument>
130-
<argument>za.co.absa.spline.model.expr.Alias</argument>
131-
<argument>za.co.absa.spline.model.expr.Binary</argument>
132-
<argument>za.co.absa.spline.model.expr.UDF</argument>
133-
<argument>za.co.absa.spline.model.op.Alias</argument>
134-
<argument>za.co.absa.spline.model.op.Destination</argument>
135-
<argument>za.co.absa.spline.model.op.Filter</argument>
136-
<argument>za.co.absa.spline.model.op.Generic</argument>
137-
<argument>za.co.absa.spline.model.op.Join</argument>
138-
<argument>za.co.absa.spline.model.op.Projection</argument>
139-
<argument>za.co.absa.spline.model.op.Source</argument>
140-
<argument>za.co.absa.spline.model.dt.DataType</argument>
141-
<argument>za.co.absa.spline.model.dt.Simple</argument>
142-
<argument>za.co.absa.spline.model.dt.Struct</argument>
143-
<argument>za.co.absa.spline.model.dt.Array</argument>
144-
</arguments>
145-
</configuration>
146219
<dependencies>
147220
<dependency>
148221
<groupId>com.github.wajda</groupId>

0 commit comments

Comments
 (0)