Merge pull request #6 from poligonoio/feature/ds-ref

fix: add support for multiple data source type starting with MySQL
poligonoio · Jul 29, 2024 · 92333bf · 92333bf
2 parents 169b6cf + 1593467
commit 92333bf
Show file tree

Hide file tree

Showing 19 changed files with 488 additions and 352 deletions.
diff --git a/README.md b/README.md
@@ -34,6 +34,11 @@ Poligono is designed to run in a containerized environment. Here's the simplest
 
 Once the Docker installation is complete, go to [http://localhost:8888/v1/swagger/index.html#/](http://localhost:8888/v1/swagger/index.html#/) to access the Poligono OpenAPI Specification from your browser.
 
+## 💾 Supported Data Sources
+
+* PostgreSQL
+* MySQL
+
 <!-- CONTRIBUTING -->
 ## Contributing
 

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -28,3 +28,11 @@ services:
       CATALOG_MANAGEMENT: dynamic
     ports:
     - "8080:8080"
+
+  db:
+    image: mysql
+    restart: always
+    ports:
+    - "3306:3306"
+    environment:
+      MYSQL_ROOT_PASSWORD: poligono
diff --git a/docs/docs.go b/docs/docs.go
@@ -379,7 +379,8 @@ const docTemplate = `{
                 },
                 "type": {
                     "enum": [
-                        "PostgreSQL"
+                        "PostgreSQL",
+                        "MySQL"
                     ],
                     "allOf": [
                         {
@@ -392,10 +393,12 @@ const docTemplate = `{
         "models.DataSourceType": {
             "type": "string",
             "enum": [
-                "PostgreSQL"
+                "PostgreSQL",
+                "MySQL"
             ],
             "x-enum-varnames": [
-                "PostgreSQL"
+                "PostgreSQL",
+                "MySQL"
             ]
         },
         "models.GenerateQueryActivity": {
@@ -504,7 +507,8 @@ const docTemplate = `{
                 },
                 "type": {
                     "enum": [
-                        "PostgreSQL"
+                        "PostgreSQL",
+                        "MySQL"
                     ],
                     "allOf": [
                         {

diff --git a/docs/swagger.json b/docs/swagger.json
@@ -370,7 +370,8 @@
                 },
                 "type": {
                     "enum": [
-                        "PostgreSQL"
+                        "PostgreSQL",
+                        "MySQL"
                     ],
                     "allOf": [
                         {
@@ -383,10 +384,12 @@
         "models.DataSourceType": {
             "type": "string",
             "enum": [
-                "PostgreSQL"
+                "PostgreSQL",
+                "MySQL"
             ],
             "x-enum-varnames": [
-                "PostgreSQL"
+                "PostgreSQL",
+                "MySQL"
             ]
         },
         "models.GenerateQueryActivity": {
@@ -495,7 +498,8 @@
                 },
                 "type": {
                     "enum": [
-                        "PostgreSQL"
+                        "PostgreSQL",
+                        "MySQL"
                     ],
                     "allOf": [
                         {

diff --git a/docs/swagger.yaml b/docs/swagger.yaml
@@ -10,6 +10,7 @@ definitions:
         - $ref: '#/definitions/models.DataSourceType'
         enum:
         - PostgreSQL
+        - MySQL
     required:
     - name
     - secret
@@ -18,9 +19,11 @@ definitions:
   models.DataSourceType:
     enum:
     - PostgreSQL
+    - MySQL
     type: string
     x-enum-varnames:
     - PostgreSQL
+    - MySQL
   models.GenerateQueryActivity:
     properties:
       data:
@@ -94,6 +97,7 @@ definitions:
         - $ref: '#/definitions/models.DataSourceType'
         enum:
         - PostgreSQL
+        - MySQL
     type: object
 externalDocs:
   description: OpenAPI

diff --git a/internal/controllers/core.controllers.go b/internal/controllers/core.controllers.go
@@ -16,10 +16,10 @@ import (
 type CoreController struct {
 	CoreService       services.CoreService
 	DataSourceService services.DataSourceService
-	TrinoService      services.TrinoService
+	TrinoService      services.EngineService
 }
 
-func NewCoreController(coreService services.CoreService, dataSourceService services.DataSourceService, trinoService services.TrinoService) CoreController {
+func NewCoreController(coreService services.CoreService, dataSourceService services.DataSourceService, trinoService services.EngineService) CoreController {
 	return CoreController{
 		CoreService:       coreService,
 		DataSourceService: dataSourceService,

diff --git a/internal/controllers/ds.controllers.go b/internal/controllers/ds.controllers.go
@@ -16,12 +16,12 @@ import (
 
 type DataSourceController struct {
 	DataSourceService services.DataSourceService
-	TrinoService      services.TrinoService
+	TrinoService      services.EngineService
 	SchemaService     services.SchemaService
 	validate          *validatorv10.Validate
 }
 
-func NewDataSourceController(dataSourceService services.DataSourceService, trinoService services.TrinoService, schemaService services.SchemaService, validate *validatorv10.Validate) DataSourceController {
+func NewDataSourceController(dataSourceService services.DataSourceService, trinoService services.EngineService, schemaService services.SchemaService, validate *validatorv10.Validate) DataSourceController {
 	return DataSourceController{
 		DataSourceService: dataSourceService,
 		TrinoService:      trinoService,

diff --git a/internal/models/datasource.go b/internal/models/datasource.go
@@ -1,10 +1,6 @@
 package models
 
 import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"strconv"
 	"time"
 
 	"go.mongodb.org/mongo-driver/bson/primitive"
@@ -14,27 +10,35 @@ type DataSourceType string
 
 const (
 	PostgreSQL DataSourceType = "PostgreSQL"
+	MySQL      DataSourceType = "MySQL"
+)
+
+type EngineType string
+
+const (
+	Trino     EngineType = "Trino"
+	Starburst EngineType = "Starburst"
 )
 
 type DataSource struct {
 	ID             primitive.ObjectID `json:"-" bson:"omitempty,_id" swaggerignore:"true"`
 	Name           string             `json:"name"  bson:"name" validate:"required"`
 	OrganizationId string             `json:"organization_id" bson:"organization_id" validate:"required" swaggerignore:"true"`
 	CreatedBy      string             `json:"-" bson:"created_by" validate:"required"`
-	Type           DataSourceType     `json:"type" bson:"type" validate:"required,oneof=PostgreSQL"`
+	Type           DataSourceType     `json:"type" bson:"type" validate:"required,oneof=PostgreSQL MySQL"`
 	Secret         string             `json:"secret,omitempty" bson:"-" validate:"required"`
 	CreatedAt      time.Time          `json:"created_at" bson:"created_at" validate:"required" swaggerignore:"true"`
 	UpdatedAt      time.Time          `json:"updated_at" bson:"updated_at" validate:"required" swaggerignore:"true"`
 }
 
 type UpdateRequestDataSourceBody struct {
 	Name      string         `json:"name"  bson:"name"`
-	Type      DataSourceType `json:"type" bson:"type" validate:"omitempty,oneof=PostgreSQL"`
+	Type      DataSourceType `json:"type" bson:"type" validate:"omitempty,oneof=PostgreSQL MySQL"`
 	Secret    string         `json:"secret" bson:"-"`
 	UpdatedAt time.Time      `json:"-" bson:"updated_at"`
 }
 
-type PostgreSQLObject struct {
+type PostgreSQLSecret struct {
 	Host     string `json:"hostname"`
 	Port     int    `json:"port"`
 	User     string `json:"username"`
@@ -43,25 +47,11 @@ type PostgreSQLObject struct {
 	SSL      bool   `json:"ssl"`
 }
 
-func GetCatalogQuery(catalogName string, dataSourceType DataSourceType, secret string) (string, error) {
-	var err error
-	switch dataSourceType {
-	case PostgreSQL:
-		psql := PostgreSQLObject{}
-		if err = json.Unmarshal([]byte(secret), &psql); err != nil {
-			return "", err
-		}
-
-		var psqlString string
-
-		if psql.SSL {
-			psqlString = fmt.Sprintf("jdbc:postgresql://%s:%s/%s?sslmode=require", psql.Host, strconv.Itoa(psql.Port), psql.Database)
-		} else {
-			psqlString = fmt.Sprintf("jdbc:postgresql://%s:%s/%s", psql.Host, strconv.Itoa(psql.Port), psql.Database)
-		}
-
-		return fmt.Sprintf("CREATE CATALOG %s USING postgresql WITH (\"connection-url\" = '%s', \"connection-user\" = '%s', \"connection-password\" = '%s', \"case-insensitive-name-matching\" = 'true', \"postgresql.include-system-tables\" = 'true')", catalogName, psqlString, psql.User, psql.Password), nil
-	default:
-		return "", errors.New("invalid data source type")
-	}
+type MySQLSecret struct {
+	Host     string `json:"hostname"`
+	Port     int    `json:"port"`
+	User     string `json:"username"`
+	Database string `json:"database"`
+	Password string `json:"password"`
+	SSL      bool   `json:"ssl"`
 }
diff --git a/internal/models/schema.go b/internal/models/schema.go
@@ -4,8 +4,8 @@ import (
 	"go.mongodb.org/mongo-driver/bson/primitive"
 )
 
-type PSQLSchema struct {
-	Nspname string `json:"nspname"`
+type SQLSchema struct {
+	Name string `json:"name"`
 }
 
 type Schemas struct {
@@ -28,9 +28,8 @@ type UpdateSchema struct {
 	Tables         []Table `json:"tables" bson:"tables"`
 }
 
-type PSQLTable struct {
-	Relname      string `json:"relname"`
-	Relnamespace int64  `json:"relnamespace"`
+type SQLTable struct {
+	Name string `json:"name"`
 }
 
 type Table struct {
@@ -39,9 +38,8 @@ type Table struct {
 	Fields      []Field `json:"fields" bson:"fields"`
 }
 
-type PSQLField struct {
-	Attname  string `json:"attname"`
-	Attrelid int    `json:"attrelid"`
+type SQLField struct {
+	Name string `json:"name"`
 }
 
 type Field struct {

diff --git a/internal/services/ds.service.go b/internal/services/ds.service.go
@@ -11,5 +11,4 @@ type DataSourceService interface {
 	Update(name string, organizationId string, newDataSource models.UpdateRequestDataSourceBody) error
 	Delete(name string, organizationId string) error
 	GetDataSourceSchemas(dataSourceName string, organizationId string) ([]models.Schema, error)
-	Sync(dataSourceName string, organizationId string) error
 }